From 668323c39fd6afbe6c2e1d87b6fd0ab4ae8516f2 Mon Sep 17 00:00:00 2001 From: James Cor Date: Fri, 20 Sep 2024 11:29:35 -0700 Subject: [PATCH 01/39] small refactor --- server/handler.go | 21 +++++++++------------ sql/rowexec/rel_iters.go | 13 ++++++------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/server/handler.go b/server/handler.go index 297bcedc11..8b499499a1 100644 --- a/server/handler.go +++ b/server/handler.go @@ -552,19 +552,23 @@ func (h *Handler) resultForDefaultIter( eg, ctx := ctx.NewErrgroup() - var rowChan chan sql.Row - - rowChan = make(chan sql.Row, 512) - pan2err := func() { if recoveredPanic := recover(); recoveredPanic != nil { returnErr = fmt.Errorf("handler caught panic: %v", recoveredPanic) } } + pollCtx, cancelF := ctx.NewSubContext() + eg.Go(func() error { + defer pan2err() + return h.pollForClosedConnection(pollCtx, c) + }) + wg := sync.WaitGroup{} wg.Add(2) + // Read rows off the row iterator and send them to the row channel. + var rowChan = make(chan sql.Row, 512) eg.Go(func() error { defer pan2err() defer wg.Done() @@ -590,12 +594,6 @@ func (h *Handler) resultForDefaultIter( } }) - pollCtx, cancelF := ctx.NewSubContext() - eg.Go(func() error { - defer pan2err() - return h.pollForClosedConnection(pollCtx, c) - }) - // Default waitTime is one minute if there is no timeout configured, in which case // it will loop to iterate again unless the socket died by the OS timeout or other problems. // If there is a timeout, it will be enforced to ensure that Vitess has a chance to @@ -607,7 +605,7 @@ func (h *Handler) resultForDefaultIter( timer := time.NewTimer(waitTime) defer timer.Stop() - // reads rows from the channel, converts them to wire format, + // Reads rows from the channel, converts them to wire format, // and calls |callback| to give them to vitess. eg.Go(func() error { defer pan2err() @@ -679,7 +677,6 @@ func (h *Handler) resultForDefaultIter( } returnErr = err } - return } diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index fdcb3f9fc9..9ace16e176 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -527,8 +527,8 @@ func ProjectRow( projections []sql.Expression, row sql.Row, ) (sql.Row, error) { - var secondPass []int - var fields sql.Row + var fields = make(sql.Row, len(projections)) + var secondPass = make([]int, 0, len(projections)) for i, expr := range projections { // Default values that are expressions may reference other fields, thus they must evaluate after all other exprs. // Also default expressions may not refer to other columns that come after them if they also have a default expr. @@ -536,16 +536,15 @@ func ProjectRow( // Since literals do not reference other columns, they're evaluated on the first pass. defaultVal, isDefaultVal := defaultValFromProjectExpr(expr) if isDefaultVal && !defaultVal.IsLiteral() { - fields = append(fields, nil) secondPass = append(secondPass, i) continue } - f, fErr := expr.Eval(ctx, row) + field, fErr := expr.Eval(ctx, row) if fErr != nil { return nil, fErr } - f = normalizeNegativeZeros(f) - fields = append(fields, f) + field = normalizeNegativeZeros(field) + fields[i] = field } for _, index := range secondPass { field, err := projections[index].Eval(ctx, fields) @@ -555,7 +554,7 @@ func ProjectRow( field = normalizeNegativeZeros(field) fields[index] = field } - return sql.NewRow(fields...), nil + return fields, nil } func defaultValFromProjectExpr(e sql.Expression) (*sql.ColumnDefaultValue, bool) { From bcf8ace7b886e19c6068303476e38b22d94f7b7b Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 23 Sep 2024 11:23:45 -0700 Subject: [PATCH 02/39] something --- server/handler.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/server/handler.go b/server/handler.go index 8b499499a1..00cf8a162e 100644 --- a/server/handler.go +++ b/server/handler.go @@ -903,18 +903,19 @@ func updateMaxUsedConnectionsStatusVariable() { func rowToSQL(ctx *sql.Context, s sql.Schema, row sql.Row) ([]sqltypes.Value, error) { o := make([]sqltypes.Value, len(row)) + // need to make sure the schema is not null as some plan schema is defined as null (e.g. IfElseBlock) + if len(s) == 0 { + return o, nil + } var err error for i, v := range row { if v == nil { o[i] = sqltypes.NULL continue } - // need to make sure the schema is not null as some plan schema is defined as null (e.g. IfElseBlock) - if len(s) > 0 { - o[i], err = s[i].Type.SQL(ctx, nil, v) - if err != nil { - return nil, err - } + o[i], err = s[i].Type.SQL(ctx, nil, v) + if err != nil { + return nil, err } } From c24dc761a675380198226b5737162feea1a237f2 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 23 Sep 2024 14:53:20 -0700 Subject: [PATCH 03/39] prototype --- server/handler.go | 46 ++++++++++++++++++++++++++--------- server/server.go | 1 + sql/analyzer/analyzer.go | 2 ++ sql/analyzer/rule_ids.go | 1 + sql/analyzer/ruleid_string.go | 13 +++++----- sql/analyzer/rules.go | 1 + sql/analyzer/warnings.go | 23 +++++++++++++++++- sql/plan/process.go | 26 ++++++++++---------- sql/plan/project.go | 1 + 9 files changed, 82 insertions(+), 32 deletions(-) diff --git a/server/handler.go b/server/handler.go index 00cf8a162e..2d116be51f 100644 --- a/server/handler.go +++ b/server/handler.go @@ -528,7 +528,7 @@ func resultForMax1RowIter(ctx *sql.Context, schema sql.Schema, iter sql.RowIter, return nil, err } - outputRow, err := rowToSQL(ctx, schema, row) + outputRow, err := rowToSQL(ctx, schema, row, nil) if err != nil { return nil, err } @@ -605,6 +605,15 @@ func (h *Handler) resultForDefaultIter( timer := time.NewTimer(waitTime) defer timer.Stop() + var projections []sql.Expression + if trackedIter, ok := iter.(*plan.TrackedRowIter); ok { + if commitNode, ok := trackedIter.Node.(*plan.TransactionCommittingNode); ok { + if proj, ok := commitNode.Child().(*plan.Project); ok { + projections = proj.Projections + } + } + } + // Reads rows from the channel, converts them to wire format, // and calls |callback| to give them to vitess. eg.Go(func() error { @@ -639,7 +648,7 @@ func (h *Handler) resultForDefaultIter( continue } - outputRow, err := rowToSQL(ctx, schema, row) + outputRow, err := rowToSQL(ctx, schema, row, projections) if err != nil { return err } @@ -901,21 +910,34 @@ func updateMaxUsedConnectionsStatusVariable() { }() } -func rowToSQL(ctx *sql.Context, s sql.Schema, row sql.Row) ([]sqltypes.Value, error) { - o := make([]sqltypes.Value, len(row)) +func rowToSQL(ctx *sql.Context, s sql.Schema, row sql.Row, projections []sql.Expression) ([]sqltypes.Value, error) { + o := make([]sqltypes.Value, max(len(row), len(projections))) // TODO: maybe should be length of schema? // need to make sure the schema is not null as some plan schema is defined as null (e.g. IfElseBlock) if len(s) == 0 { return o, nil } - var err error - for i, v := range row { - if v == nil { - o[i] = sqltypes.NULL - continue + if len(projections) > 0 { + for i, proj := range projections { + field, err := proj.Eval(ctx, row) + if err != nil { + return nil, err + } + o[i], err = s[i].Type.SQL(ctx, nil, field) + if err != nil { + return nil, err + } } - o[i], err = s[i].Type.SQL(ctx, nil, v) - if err != nil { - return nil, err + } else { + var err error + for i, v := range row { + if v == nil { + o[i] = sqltypes.NULL + continue + } + o[i], err = s[i].Type.SQL(ctx, nil, v) + if err != nil { + return nil, err + } } } diff --git a/server/server.go b/server/server.go index e77a933253..12e7820aaf 100644 --- a/server/server.go +++ b/server/server.go @@ -182,6 +182,7 @@ func newServerFromHandler(cfg Config, e *sqle.Engine, sm *SessionManager, handle vtListener.RequireSecureTransport = cfg.RequireSecureTransport } + e.Analyzer.ServerMode = true return &Server{ Listener: protocolListener, handler: handler, diff --git a/sql/analyzer/analyzer.go b/sql/analyzer/analyzer.go index 8a2834ecbb..da20fa3130 100644 --- a/sql/analyzer/analyzer.go +++ b/sql/analyzer/analyzer.go @@ -298,6 +298,8 @@ type Analyzer struct { // EventScheduler is used to communiate with the event scheduler // for any EVENT related statements. It can be nil if EventScheduler is not defined. EventScheduler sql.EventScheduler + // ServerMode is true if the analyzer is running in server mode. + ServerMode bool } // NewDefault creates a default Analyzer instance with all default Rules and configuration. diff --git a/sql/analyzer/rule_ids.go b/sql/analyzer/rule_ids.go index 836e0a7c8b..8aa216c062 100644 --- a/sql/analyzer/rule_ids.go +++ b/sql/analyzer/rule_ids.go @@ -141,6 +141,7 @@ const ( cacheSubqueryResultsId // cacheSubqueryResults cacheSubqueryAliasesInJoinsId // cacheSubqueryAliasesInJoins backtickDefaulColumnValueNamesId // backtickDefaulColumnValueNames + DeferProjectionsId // deferProjections AutocommitId // addAutocommitNode TrackProcessId // trackProcess parallelizeId // parallelize diff --git a/sql/analyzer/ruleid_string.go b/sql/analyzer/ruleid_string.go index ee4fc77396..e79f469d70 100755 --- a/sql/analyzer/ruleid_string.go +++ b/sql/analyzer/ruleid_string.go @@ -135,15 +135,16 @@ func _() { _ = x[cacheSubqueryResultsId-124] _ = x[cacheSubqueryAliasesInJoinsId-125] _ = x[backtickDefaulColumnValueNamesId-126] - _ = x[AutocommitId-127] - _ = x[TrackProcessId-128] - _ = x[parallelizeId-129] - _ = x[clearWarningsId-130] + _ = x[DeferProjectionsId-127] + _ = x[AutocommitId-128] + _ = x[TrackProcessId-129] + _ = x[parallelizeId-130] + _ = x[clearWarningsId-131] } -const _RuleId_name = "applyDefaultSelectLimitvalidateOffsetAndLimitvalidateStarExpressionsvalidateCreateTablevalidateAlterTablevalidateExprSemresolveVariablesresolveNamedWindowsresolveSetVariablesresolveViewsliftCtesresolveCtesliftRecursiveCtesresolveDatabasesresolveTablesloadStoredProceduresvalidateDropTablespruneDropTablessetTargetSchemasresolveCreateLikeparseColumnDefaultsresolveDropConstraintvalidateDropConstraintloadCheckConstraintsassignCatalogresolveAnalyzeTablesresolveCreateSelectresolveSubqueriessetViewTargetSchemaresolveUnionsresolveDescribeQuerycheckUniqueTableNamesresolveTableFunctionsresolveDeclarationsresolveColumnDefaultsValidateColumnDefaultsvalidateCreateTriggervalidateCreateProcedureresolveCreateProcedureloadInfoSchemavalidateReadOnlyDatabasevalidateReadOnlyTransactionvalidateDatabaseSetvalidatePrivilegesreresolveTablessetInsertColumnsvalidateJoinComplexityapplyBinlogReplicaControllerapplyEventSchedulerresolveUsingJoinsresolveOrderbyLiteralsresolveFunctionsflattenTableAliasespushdownSortpushdownGroupbyAliasespushdownSubqueryAliasFiltersqualifyColumnsresolveColumnsvalidateCheckConstraintresolveBarewordSetVariablesreplaceCountStarexpandStarstransposeRightJoinsresolveHavingmergeUnionSchemasflattenAggregationExprsreorderProjectionresolveSubqueryExprsreplaceCrossJoinsmoveJoinCondsToFiltermoveFiltersToJoinCondsimplifyFilterspushNotFiltersoptimizeDistincthoistOutOfScopeFiltersunnestInSubqueriesunnestExistsSubqueriesfinalizeSubqueriesfinalizeUnionsloadTriggersloadEventsprocessTruncateresolveAlterColumnresolveGeneratorsremoveUnnecessaryConvertsstripTableNamesFromColumnDefaultsfoldEmptyJoinsoptimizeJoinsgenerateIndexScansmatchAgainstpushFiltersapplyIndexesFromOuterScopepruneTablesfixupAuxiliaryExprsassignExecIndexesinlineSubqueryAliasRefseraseProjectionflattenDistinctreplaceAggreplaceIdxSortinsertTopNapplyHashInresolveInsertRowsresolvePreparedInsertapplyTriggersapplyProceduresassignRoutinesmodifyUpdateExprsForJoinapplyRowUpdateAccumulatorsrollback triggersapplyFKsvalidateResolvedvalidateOrderByvalidateGroupByvalidateSchemaSourcevalidateIndexCreationValidateOperandsvalidateCaseResultTypesvalidateIntervalUsagevalidateExplodeUsagevalidateSubqueryColumnsvalidateUnionSchemasMatchvalidateAggregationsvalidateDeleteFromcacheSubqueryResultscacheSubqueryAliasesInJoinsbacktickDefaulColumnValueNamesaddAutocommitNodetrackProcessparallelizeclearWarnings" +const _RuleId_name = "applyDefaultSelectLimitvalidateOffsetAndLimitvalidateStarExpressionsvalidateCreateTablevalidateAlterTablevalidateExprSemresolveVariablesresolveNamedWindowsresolveSetVariablesresolveViewsliftCtesresolveCtesliftRecursiveCtesresolveDatabasesresolveTablesloadStoredProceduresvalidateDropTablespruneDropTablessetTargetSchemasresolveCreateLikeparseColumnDefaultsresolveDropConstraintvalidateDropConstraintloadCheckConstraintsassignCatalogresolveAnalyzeTablesresolveCreateSelectresolveSubqueriessetViewTargetSchemaresolveUnionsresolveDescribeQuerycheckUniqueTableNamesresolveTableFunctionsresolveDeclarationsresolveColumnDefaultsvalidateColumnDefaultsvalidateCreateTriggervalidateCreateProcedureresolveCreateProcedureloadInfoSchemavalidateReadOnlyDatabasevalidateReadOnlyTransactionvalidateDatabaseSetvalidatePrivilegesreresolveTablessetInsertColumnsvalidateJoinComplexityapplyBinlogReplicaControllerapplyEventSchedulerresolveUsingJoinsresolveOrderbyLiteralsresolveFunctionsflattenTableAliasespushdownSortpushdownGroupbyAliasespushdownSubqueryAliasFiltersqualifyColumnsresolveColumnsvalidateCheckConstraintresolveBarewordSetVariablesreplaceCountStarexpandStarstransposeRightJoinsresolveHavingmergeUnionSchemasflattenAggregationExprsreorderProjectionresolveSubqueryExprsreplaceCrossJoinsmoveJoinCondsToFiltermoveFiltersToJoinCondsimplifyFilterspushNotFiltersoptimizeDistincthoistOutOfScopeFiltersunnestInSubqueriesunnestExistsSubqueriesfinalizeSubqueriesfinalizeUnionsloadTriggersloadEventsprocessTruncateresolveAlterColumnresolveGeneratorsremoveUnnecessaryConvertsstripTableNamesFromColumnDefaultsfoldEmptyJoinsoptimizeJoinsgenerateIndexScansmatchAgainstpushFiltersapplyIndexesFromOuterScopepruneTablesfixupAuxiliaryExprsassignExecIndexesinlineSubqueryAliasRefseraseProjectionflattenDistinctreplaceAggreplaceIdxSortinsertTopNapplyHashInresolveInsertRowsresolvePreparedInsertapplyTriggersapplyProceduresassignRoutinesmodifyUpdateExprsForJoinapplyRowUpdateAccumulatorsrollback triggersapplyFKsvalidateResolvedvalidateOrderByvalidateGroupByvalidateSchemaSourcevalidateIndexCreationvalidateOperandsvalidateCaseResultTypesvalidateIntervalUsagevalidateExplodeUsagevalidateSubqueryColumnsvalidateUnionSchemasMatchvalidateAggregationsvalidateDeleteFromcacheSubqueryResultscacheSubqueryAliasesInJoinsbacktickDefaulColumnValueNamesdeferProjectionsaddAutocommitNodetrackProcessparallelizeclearWarnings" -var _RuleId_index = [...]uint16{0, 23, 45, 68, 87, 105, 120, 136, 155, 174, 186, 194, 205, 222, 238, 251, 271, 289, 304, 320, 337, 356, 377, 399, 419, 432, 452, 471, 488, 507, 520, 540, 561, 582, 601, 622, 644, 665, 688, 710, 724, 748, 775, 794, 812, 827, 843, 865, 893, 912, 929, 951, 967, 986, 998, 1020, 1048, 1062, 1076, 1099, 1126, 1142, 1153, 1172, 1185, 1202, 1225, 1242, 1262, 1279, 1300, 1321, 1336, 1350, 1366, 1388, 1406, 1428, 1446, 1460, 1472, 1482, 1497, 1515, 1532, 1557, 1590, 1604, 1617, 1635, 1647, 1658, 1684, 1695, 1714, 1731, 1754, 1769, 1784, 1794, 1808, 1818, 1829, 1846, 1867, 1880, 1895, 1909, 1933, 1959, 1976, 1984, 2000, 2015, 2030, 2050, 2071, 2087, 2110, 2131, 2151, 2174, 2199, 2219, 2237, 2257, 2284, 2314, 2331, 2343, 2354, 2367} +var _RuleId_index = [...]uint16{0, 23, 45, 68, 87, 105, 120, 136, 155, 174, 186, 194, 205, 222, 238, 251, 271, 289, 304, 320, 337, 356, 377, 399, 419, 432, 452, 471, 488, 507, 520, 540, 561, 582, 601, 622, 644, 665, 688, 710, 724, 748, 775, 794, 812, 827, 843, 865, 893, 912, 929, 951, 967, 986, 998, 1020, 1048, 1062, 1076, 1099, 1126, 1142, 1153, 1172, 1185, 1202, 1225, 1242, 1262, 1279, 1300, 1321, 1336, 1350, 1366, 1388, 1406, 1428, 1446, 1460, 1472, 1482, 1497, 1515, 1532, 1557, 1590, 1604, 1617, 1635, 1647, 1658, 1684, 1695, 1714, 1731, 1754, 1769, 1784, 1794, 1808, 1818, 1829, 1846, 1867, 1880, 1895, 1909, 1933, 1959, 1976, 1984, 2000, 2015, 2030, 2050, 2071, 2087, 2110, 2131, 2151, 2174, 2199, 2219, 2237, 2257, 2284, 2314, 2330, 2347, 2359, 2370, 2383} func (i RuleId) String() string { if i < 0 || i >= RuleId(len(_RuleId_index)-1) { diff --git a/sql/analyzer/rules.go b/sql/analyzer/rules.go index 8e73c6e3a9..5d3f3dc3e4 100644 --- a/sql/analyzer/rules.go +++ b/sql/analyzer/rules.go @@ -26,6 +26,7 @@ func init() { {inlineSubqueryAliasRefsId, inlineSubqueryAliasRefs}, {cacheSubqueryAliasesInJoinsId, cacheSubqueryAliasesInJoins}, {backtickDefaulColumnValueNamesId, backtickDefaultColumnValueNames}, + {DeferProjectionsId, deferProjections}, {AutocommitId, addAutocommitNode}, {TrackProcessId, trackProcess}, {parallelizeId, parallelize}, diff --git a/sql/analyzer/warnings.go b/sql/analyzer/warnings.go index 87807eb377..58e741ba37 100644 --- a/sql/analyzer/warnings.go +++ b/sql/analyzer/warnings.go @@ -16,7 +16,8 @@ package analyzer import ( "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/plan" + "github.com/dolthub/go-mysql-server/sql/expression" +"github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/transform" ) @@ -40,3 +41,23 @@ func clearWarnings(ctx *sql.Context, a *Analyzer, node sql.Node, scope *plan.Sco ctx.ClearWarnings() return node, transform.SameTree, nil } + +// TODO: move this somewhere else +func deferProjections(ctx *sql.Context, a *Analyzer, node sql.Node, scope *plan.Scope, sel RuleSelector, qFlags *sql.QueryFlags) (sql.Node, transform.TreeIdentity, error) { + if !a.ServerMode { + return node, transform.SameTree, nil + } + // Find top-level projection, and mark as deferred + if proj, ok := node.(*plan.Project); ok { + // Default value expressions require a second pass, so punt on deferring for now + for _, expr := range proj.Projections { + switch expr.(type) { + case *expression.Wrapper, *sql.ColumnDefaultValue: + return node, transform.SameTree, nil + } + } + proj.Deferred = true + return proj, transform.NewTree, nil + } + return node, transform.SameTree, nil +} \ No newline at end of file diff --git a/sql/plan/process.go b/sql/plan/process.go index 537bc1f40d..b154c03e40 100644 --- a/sql/plan/process.go +++ b/sql/plan/process.go @@ -306,8 +306,8 @@ const ( QueryTypeUpdate ) -type trackedRowIter struct { - node sql.Node +type TrackedRowIter struct { + Node sql.Node iter sql.RowIter numRows int64 QueryType queryType @@ -321,18 +321,18 @@ func NewTrackedRowIter( iter sql.RowIter, onNext NotifyFunc, onDone NotifyFunc, -) *trackedRowIter { - return &trackedRowIter{node: node, iter: iter, onDone: onDone, onNext: onNext} +) *TrackedRowIter { + return &TrackedRowIter{Node: node, iter: iter, onDone: onDone, onNext: onNext} } -func (i *trackedRowIter) done() { +func (i *TrackedRowIter) done() { if i.onDone != nil { i.onDone() i.onDone = nil } - if i.node != nil { + if i.Node != nil { i.Dispose() - i.node = nil + i.Node = nil } } @@ -347,13 +347,13 @@ func disposeNode(n sql.Node) { }) } -func (i *trackedRowIter) Dispose() { - if i.node != nil { - disposeNode(i.node) +func (i *TrackedRowIter) Dispose() { + if i.Node != nil { + disposeNode(i.Node) } } -func (i *trackedRowIter) Next(ctx *sql.Context) (sql.Row, error) { +func (i *TrackedRowIter) Next(ctx *sql.Context) (sql.Row, error) { row, err := i.iter.Next(ctx) if err != nil { return nil, err @@ -368,7 +368,7 @@ func (i *trackedRowIter) Next(ctx *sql.Context) (sql.Row, error) { return row, nil } -func (i *trackedRowIter) Close(ctx *sql.Context) error { +func (i *TrackedRowIter) Close(ctx *sql.Context) error { err := i.iter.Close(ctx) i.updateSessionVars(ctx) @@ -377,7 +377,7 @@ func (i *trackedRowIter) Close(ctx *sql.Context) error { return err } -func (i *trackedRowIter) updateSessionVars(ctx *sql.Context) { +func (i *TrackedRowIter) updateSessionVars(ctx *sql.Context) { switch i.QueryType { case QueryTypeSelect: ctx.SetLastQueryInfoInt(sql.RowCount, -1) diff --git a/sql/plan/project.go b/sql/plan/project.go index 3ef980e671..9786b78051 100644 --- a/sql/plan/project.go +++ b/sql/plan/project.go @@ -28,6 +28,7 @@ type Project struct { UnaryNode // Expression projected. Projections []sql.Expression + Deferred bool } var _ sql.Expressioner = (*Project)(nil) From 9cfa2f7f5e7d3425056c450943a000fe36bc1011 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 23 Sep 2024 15:03:00 -0700 Subject: [PATCH 04/39] fix defer --- sql/plan/project.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sql/plan/project.go b/sql/plan/project.go index 9786b78051..794db66f86 100644 --- a/sql/plan/project.go +++ b/sql/plan/project.go @@ -162,7 +162,9 @@ func (p *Project) WithChildren(children ...sql.Node) (sql.Node, error) { return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 1) } - return NewProject(p.Projections, children[0]), nil + np := NewProject(p.Projections, children[0]) + np.Deferred = p.Deferred + return np, nil } // CheckPrivileges implements the interface sql.Node. @@ -181,5 +183,7 @@ func (p *Project) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { return nil, sql.ErrInvalidChildrenNumber.New(p, len(exprs), len(p.Projections)) } - return NewProject(exprs, p.Child), nil + np := NewProject(exprs, p.Child) + np.Deferred = p.Deferred + return np, nil } From 558c0b518a7084ab211bee03852cee74c76c1bff Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 23 Sep 2024 15:26:31 -0700 Subject: [PATCH 05/39] sdf --- sql/rowexec/dml.go | 2 +- sql/rowexec/rel.go | 9 +++++---- sql/rowexec/rel_iters.go | 15 +++++++++------ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/sql/rowexec/dml.go b/sql/rowexec/dml.go index b10c9c4044..4c86bf17b5 100644 --- a/sql/rowexec/dml.go +++ b/sql/rowexec/dml.go @@ -375,7 +375,7 @@ func (b *BaseBuilder) buildRowUpdateAccumulator(ctx *sql.Context, n *plan.RowUpd case *updateJoinIter: i.accumulator = rowHandler.(*updateJoinRowHandler) done = true - case *projectIter: + case *ProjectIter: iter = i.childIter case *plan.CheckpointingTableEditorIter: iter = i.InnerIter() diff --git a/sql/rowexec/rel.go b/sql/rowexec/rel.go index 6b0e6f522e..510b03ed68 100644 --- a/sql/rowexec/rel.go +++ b/sql/rowexec/rel.go @@ -310,8 +310,9 @@ func (b *BaseBuilder) buildProject(ctx *sql.Context, n *plan.Project, row sql.Ro return nil, err } - return sql.NewSpanIter(span, &projectIter{ - p: n.Projections, + return sql.NewSpanIter(span, &ProjectIter{ + Projs: n.Projections, + deferred: n.Deferred, childIter: i, }), nil } @@ -321,8 +322,8 @@ func (b *BaseBuilder) buildVirtualColumnTable(ctx *sql.Context, n *plan.VirtualC attribute.Int("projections", len(n.Projections)), )) - return sql.NewSpanIter(span, &projectIter{ - p: n.Projections, + return sql.NewSpanIter(span, &ProjectIter{ + Projs: n.Projections, childIter: tableIter, }), nil } diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index 9ace16e176..0b9e66de9a 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -503,21 +503,24 @@ func (di *orderedDistinctIter) Close(ctx *sql.Context) error { return di.childIter.Close(ctx) } -type projectIter struct { - p []sql.Expression +type ProjectIter struct { + Projs []sql.Expression + deferred bool childIter sql.RowIter } -func (i *projectIter) Next(ctx *sql.Context) (sql.Row, error) { +func (i *ProjectIter) Next(ctx *sql.Context) (sql.Row, error) { childRow, err := i.childIter.Next(ctx) if err != nil { return nil, err } - - return ProjectRow(ctx, i.p, childRow) + if i.deferred { + return childRow, nil + } + return ProjectRow(ctx, i.Projs, childRow) } -func (i *projectIter) Close(ctx *sql.Context) error { +func (i *ProjectIter) Close(ctx *sql.Context) error { return i.childIter.Close(ctx) } From dabce47538bc0fc5afc7578aaac20d5577ffd3f0 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 23 Sep 2024 15:32:26 -0700 Subject: [PATCH 06/39] only defer --- server/handler.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/handler.go b/server/handler.go index 2d116be51f..fc731b9042 100644 --- a/server/handler.go +++ b/server/handler.go @@ -609,7 +609,9 @@ func (h *Handler) resultForDefaultIter( if trackedIter, ok := iter.(*plan.TrackedRowIter); ok { if commitNode, ok := trackedIter.Node.(*plan.TransactionCommittingNode); ok { if proj, ok := commitNode.Child().(*plan.Project); ok { - projections = proj.Projections + if proj.Deferred { + projections = proj.Projections + } } } } From 5b1d37cf331296c4511b3613278ff40db0ebf93d Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 23 Sep 2024 16:12:41 -0700 Subject: [PATCH 07/39] fix enginetest --- enginetest/enginetests.go | 20 +++++++++++++++----- enginetest/server_engine.go | 1 + server/handler.go | 6 +++--- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/enginetest/enginetests.go b/enginetest/enginetests.go index f5d5613c04..33f44aa70f 100644 --- a/enginetest/enginetests.go +++ b/enginetest/enginetests.go @@ -5636,6 +5636,16 @@ func TestTypesOverWire(t *testing.T, harness ClientHarness, sessionBuilder serve require.NoError(t, err) expectedRowSet := script.Results[queryIdx] expectedRowIdx := 0 + var projections []sql.Expression + if trackedIter, ok := engineIter.(*plan.TrackedRowIter); ok { + if commitNode, ok := trackedIter.Node.(*plan.TransactionCommittingNode); ok { + if proj, ok := commitNode.Child().(*plan.Project); ok { + if proj.Deferred { + projections = proj.Projections + } + } + } + } var engineRow sql.Row for engineRow, err = engineIter.Next(ctx); err == nil; engineRow, err = engineIter.Next(ctx) { if !assert.True(t, r.Next()) { @@ -5653,11 +5663,11 @@ func TestTypesOverWire(t *testing.T, harness ClientHarness, sessionBuilder serve break } expectedEngineRow := make([]*string, len(engineRow)) - for i := range engineRow { - sqlVal, err := sch[i].Type.SQL(ctx, nil, engineRow[i]) - if !assert.NoError(t, err) { - break - } + row, err := server.RowToSQL(ctx, sch, engineRow, projections) + if !assert.NoError(t, err) { + break + } + for i, sqlVal := range row { if !sqlVal.IsNull() { str := sqlVal.ToString() expectedEngineRow[i] = &str diff --git a/enginetest/server_engine.go b/enginetest/server_engine.go index da7f22e03a..8f59429ee9 100644 --- a/enginetest/server_engine.go +++ b/enginetest/server_engine.go @@ -78,6 +78,7 @@ func NewServerQueryEngine(t *testing.T, engine *sqle.Engine, builder server.Sess _ = s.Start() }() + engine.Analyzer.ServerMode = true return &ServerQueryEngine{ t: t, engine: engine, diff --git a/server/handler.go b/server/handler.go index fc731b9042..cc87c46062 100644 --- a/server/handler.go +++ b/server/handler.go @@ -528,7 +528,7 @@ func resultForMax1RowIter(ctx *sql.Context, schema sql.Schema, iter sql.RowIter, return nil, err } - outputRow, err := rowToSQL(ctx, schema, row, nil) + outputRow, err := RowToSQL(ctx, schema, row, nil) if err != nil { return nil, err } @@ -650,7 +650,7 @@ func (h *Handler) resultForDefaultIter( continue } - outputRow, err := rowToSQL(ctx, schema, row, projections) + outputRow, err := RowToSQL(ctx, schema, row, projections) if err != nil { return err } @@ -912,7 +912,7 @@ func updateMaxUsedConnectionsStatusVariable() { }() } -func rowToSQL(ctx *sql.Context, s sql.Schema, row sql.Row, projections []sql.Expression) ([]sqltypes.Value, error) { +func RowToSQL(ctx *sql.Context, s sql.Schema, row sql.Row, projections []sql.Expression) ([]sqltypes.Value, error) { o := make([]sqltypes.Value, max(len(row), len(projections))) // TODO: maybe should be length of schema? // need to make sure the schema is not null as some plan schema is defined as null (e.g. IfElseBlock) if len(s) == 0 { From 72b403c73c4a4d9a9a361101187327c7a83046aa Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 23 Sep 2024 16:46:12 -0700 Subject: [PATCH 08/39] fix enginetests --- enginetest/engine_only_test.go | 1 + enginetest/memory_engine_test.go | 12 ++++++++++-- server/handler.go | 9 +++++++-- sql/rowexec/expr_closer.go | 20 ++++++++++---------- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/enginetest/engine_only_test.go b/enginetest/engine_only_test.go index 6fb9c344a7..0e6ae38c2d 100644 --- a/enginetest/engine_only_test.go +++ b/enginetest/engine_only_test.go @@ -790,6 +790,7 @@ func TestCollationCoercion(t *testing.T) { func TestRegex(t *testing.T) { harness := enginetest.NewDefaultMemoryHarness() harness.Setup(setup.SimpleSetup...) + harness.UseServer() engine, err := harness.NewEngine(t) require.NoError(t, err) defer engine.Close() diff --git a/enginetest/memory_engine_test.go b/enginetest/memory_engine_test.go index 204296e865..f88a6e09e2 100644 --- a/enginetest/memory_engine_test.go +++ b/enginetest/memory_engine_test.go @@ -205,17 +205,25 @@ func newUpdateResult(matched, updated int) types.OkResult { // Convenience test for debugging a single query. Unskip and set to the desired query. func TestSingleScript(t *testing.T) { - t.Skip() + //t.Skip() var scripts = []queries.ScriptTest{ { Name: "test script", SetUpScript: []string{}, - Assertions: []queries.ScriptTestAssertion{}, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "SELECT REGEXP_LIKE('testing', 'TESTING');", + Expected: []sql.Row{ + {0}, + }, + }, + }, }, } for _, test := range scripts { harness := enginetest.NewMemoryHarness("", 1, testNumPartitions, true, nil) + harness.UseServer() engine, err := harness.NewEngine(t) if err != nil { panic(err) diff --git a/server/handler.go b/server/handler.go index cc87c46062..a90135bc02 100644 --- a/server/handler.go +++ b/server/handler.go @@ -18,7 +18,8 @@ import ( "context" "encoding/base64" "fmt" - "io" + "github.com/dolthub/go-mysql-server/sql/rowexec" +"io" "net" "regexp" "runtime/trace" @@ -606,7 +607,11 @@ func (h *Handler) resultForDefaultIter( defer timer.Stop() var projections []sql.Expression - if trackedIter, ok := iter.(*plan.TrackedRowIter); ok { + iiter := iter + if exprIter, ok := iter.(*rowexec.ExprCloserIter); ok { + iiter = exprIter.Iter + } + if trackedIter, ok := iiter.(*plan.TrackedRowIter); ok { if commitNode, ok := trackedIter.Node.(*plan.TransactionCommittingNode); ok { if proj, ok := commitNode.Child().(*plan.Project); ok { if proj.Deferred { diff --git a/sql/rowexec/expr_closer.go b/sql/rowexec/expr_closer.go index b3c995fca8..1a8be52770 100644 --- a/sql/rowexec/expr_closer.go +++ b/sql/rowexec/expr_closer.go @@ -19,14 +19,14 @@ import ( "github.com/dolthub/go-mysql-server/sql/transform" ) -// exprCloserIter ensures that all expressions that implement sql.Closer are closed. This is implemented as a capturing +// ExprCloserIter ensures that all expressions that implement sql.Closer are closed. This is implemented as a capturing // iterator, as our workflow only supports closing nodes, not expressions. -type exprCloserIter struct { - iter sql.RowIter +type ExprCloserIter struct { + Iter sql.RowIter exprs []sql.Closer } -var _ sql.RowIter = (*exprCloserIter)(nil) +var _ sql.RowIter = (*ExprCloserIter)(nil) // AddExpressionCloser returns a new iterator that ensures that any expressions that implement sql.Closer are closed. // If there are no expressions that implement sql.Closer in the tree, then the original iterator is returned. @@ -43,20 +43,20 @@ func AddExpressionCloser(node sql.Node, iter sql.RowIter) sql.RowIter { if len(exprs) == 0 { return iter } - return &exprCloserIter{ - iter: iter, + return &ExprCloserIter{ + Iter: iter, exprs: exprs, } } // Next implements the interface sql.RowIter. -func (eci *exprCloserIter) Next(ctx *sql.Context) (sql.Row, error) { - return eci.iter.Next(ctx) +func (eci *ExprCloserIter) Next(ctx *sql.Context) (sql.Row, error) { + return eci.Iter.Next(ctx) } // Close implements the interface sql.RowIter. -func (eci *exprCloserIter) Close(ctx *sql.Context) error { - err := eci.iter.Close(ctx) +func (eci *ExprCloserIter) Close(ctx *sql.Context) error { + err := eci.Iter.Close(ctx) for _, expr := range eci.exprs { if nErr := expr.Close(ctx); err == nil { err = nErr From dd7161ab0318da8fb9b99d1c2818dc012c81567f Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 25 Sep 2024 16:02:52 -0700 Subject: [PATCH 09/39] tidy up --- enginetest/engine_only_test.go | 1 - enginetest/enginetests.go | 6 +-- server/handler.go | 87 +++++++++++++++--------------- sql/analyzer/optimization_rules.go | 22 ++++++++ sql/analyzer/rule_ids.go | 78 +++++++++++++-------------- sql/analyzer/rules.go | 2 +- sql/analyzer/validation_rules.go | 4 +- sql/analyzer/warnings.go | 23 +------- sql/plan/process.go | 16 +++--- sql/plan/project.go | 19 ++++--- 10 files changed, 132 insertions(+), 126 deletions(-) diff --git a/enginetest/engine_only_test.go b/enginetest/engine_only_test.go index 0e6ae38c2d..6fb9c344a7 100644 --- a/enginetest/engine_only_test.go +++ b/enginetest/engine_only_test.go @@ -790,7 +790,6 @@ func TestCollationCoercion(t *testing.T) { func TestRegex(t *testing.T) { harness := enginetest.NewDefaultMemoryHarness() harness.Setup(setup.SimpleSetup...) - harness.UseServer() engine, err := harness.NewEngine(t) require.NoError(t, err) defer engine.Close() diff --git a/enginetest/enginetests.go b/enginetest/enginetests.go index 33f44aa70f..ae6680f8de 100644 --- a/enginetest/enginetests.go +++ b/enginetest/enginetests.go @@ -5636,12 +5636,12 @@ func TestTypesOverWire(t *testing.T, harness ClientHarness, sessionBuilder serve require.NoError(t, err) expectedRowSet := script.Results[queryIdx] expectedRowIdx := 0 - var projections []sql.Expression + var projs []sql.Expression if trackedIter, ok := engineIter.(*plan.TrackedRowIter); ok { if commitNode, ok := trackedIter.Node.(*plan.TransactionCommittingNode); ok { if proj, ok := commitNode.Child().(*plan.Project); ok { if proj.Deferred { - projections = proj.Projections + projs = proj.Projections } } } @@ -5663,7 +5663,7 @@ func TestTypesOverWire(t *testing.T, harness ClientHarness, sessionBuilder serve break } expectedEngineRow := make([]*string, len(engineRow)) - row, err := server.RowToSQL(ctx, sch, engineRow, projections) + row, err := server.RowToSQL(ctx, sch, engineRow, projs) if !assert.NoError(t, err) { break } diff --git a/server/handler.go b/server/handler.go index a90135bc02..f36357c97f 100644 --- a/server/handler.go +++ b/server/handler.go @@ -18,8 +18,7 @@ import ( "context" "encoding/base64" "fmt" - "github.com/dolthub/go-mysql-server/sql/rowexec" -"io" + "io" "net" "regexp" "runtime/trace" @@ -42,6 +41,7 @@ import ( "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/analyzer" "github.com/dolthub/go-mysql-server/sql/plan" + "github.com/dolthub/go-mysql-server/sql/rowexec" "github.com/dolthub/go-mysql-server/sql/types" ) @@ -512,6 +512,23 @@ func resultForEmptyIter(ctx *sql.Context, iter sql.RowIter, resultFields []*quer return &sqltypes.Result{Fields: resultFields}, nil } +// getDeferredProjections looks for a top-level deferred projection +func getDeferredProjections(iter sql.RowIter) []sql.Expression { + switch i := iter.(type) { + case *rowexec.ExprCloserIter: + return getDeferredProjections(i.Iter) + case *plan.TrackedRowIter: + if commit, isCommit := i.GetNode().(*plan.TransactionCommittingNode); isCommit { + if proj, isProj := commit.Child().(*plan.Project); isProj { + if proj.Deferred { + return proj.Projections + } + } + } + } + return nil +} + // resultForMax1RowIter ensures that an empty iterator returns at most one row func resultForMax1RowIter(ctx *sql.Context, schema sql.Schema, iter sql.RowIter, resultFields []*querypb.Field) (*sqltypes.Result, error) { defer trace.StartRegion(ctx, "Handler.resultForMax1RowIter").End() @@ -529,7 +546,8 @@ func resultForMax1RowIter(ctx *sql.Context, schema sql.Schema, iter sql.RowIter, return nil, err } - outputRow, err := RowToSQL(ctx, schema, row, nil) + projs := getDeferredProjections(iter) + outputRow, err := RowToSQL(ctx, schema, row, projs) if err != nil { return nil, err } @@ -606,21 +624,7 @@ func (h *Handler) resultForDefaultIter( timer := time.NewTimer(waitTime) defer timer.Stop() - var projections []sql.Expression - iiter := iter - if exprIter, ok := iter.(*rowexec.ExprCloserIter); ok { - iiter = exprIter.Iter - } - if trackedIter, ok := iiter.(*plan.TrackedRowIter); ok { - if commitNode, ok := trackedIter.Node.(*plan.TransactionCommittingNode); ok { - if proj, ok := commitNode.Child().(*plan.Project); ok { - if proj.Deferred { - projections = proj.Projections - } - } - } - } - + projs := getDeferredProjections(iter) // Reads rows from the channel, converts them to wire format, // and calls |callback| to give them to vitess. eg.Go(func() error { @@ -655,7 +659,7 @@ func (h *Handler) resultForDefaultIter( continue } - outputRow, err := RowToSQL(ctx, schema, row, projections) + outputRow, err := RowToSQL(ctx, schema, row, projs) if err != nil { return err } @@ -917,38 +921,33 @@ func updateMaxUsedConnectionsStatusVariable() { }() } -func RowToSQL(ctx *sql.Context, s sql.Schema, row sql.Row, projections []sql.Expression) ([]sqltypes.Value, error) { - o := make([]sqltypes.Value, max(len(row), len(projections))) // TODO: maybe should be length of schema? +func RowToSQL(ctx *sql.Context, sch sql.Schema, row sql.Row, projs []sql.Expression) ([]sqltypes.Value, error) { // need to make sure the schema is not null as some plan schema is defined as null (e.g. IfElseBlock) - if len(s) == 0 { - return o, nil - } - if len(projections) > 0 { - for i, proj := range projections { - field, err := proj.Eval(ctx, row) - if err != nil { - return nil, err - } - o[i], err = s[i].Type.SQL(ctx, nil, field) + if len(sch) == 0 { + return []sqltypes.Value{}, nil + } + var field interface{} + var err error + outVals := make([]sqltypes.Value, len(sch)) + for i, col := range sch { + if len(projs) == 0 { + field = row[i] + } else { + field, err = projs[i].Eval(ctx, row) if err != nil { return nil, err } } - } else { - var err error - for i, v := range row { - if v == nil { - o[i] = sqltypes.NULL - continue - } - o[i], err = s[i].Type.SQL(ctx, nil, v) - if err != nil { - return nil, err - } + if field == nil { + outVals[i] = sqltypes.NULL + continue + } + outVals[i], err = col.Type.SQL(ctx, nil, field) + if err != nil { + return nil, err } } - - return o, nil + return outVals, nil } func schemaToFields(ctx *sql.Context, s sql.Schema) []*querypb.Field { diff --git a/sql/analyzer/optimization_rules.go b/sql/analyzer/optimization_rules.go index cb20012c91..bc9f515793 100644 --- a/sql/analyzer/optimization_rules.go +++ b/sql/analyzer/optimization_rules.go @@ -54,6 +54,28 @@ func eraseProjection(ctx *sql.Context, a *Analyzer, node sql.Node, scope *plan.S }) } +// deferProjections defers projections to the end of the query execution. This can avoid an unnecessary slice allocation +// specifically in the case where we are spooling rows from the server to the client. +func deferProjections(ctx *sql.Context, a *Analyzer, node sql.Node, scope *plan.Scope, sel RuleSelector, qFlags *sql.QueryFlags) (sql.Node, transform.TreeIdentity, error) { + if !a.ServerMode { + return node, transform.SameTree, nil + } + // Only defer top-level projections + proj, isProj := node.(*plan.Project) + if !isProj { + return node, transform.SameTree, nil + } + // Default value expressions require a second pass, so punt on deferring for now + for _, expr := range proj.Projections { + switch expr.(type) { + case *expression.Wrapper, *sql.ColumnDefaultValue: + return node, transform.SameTree, nil + } + } + newProj := proj.WithDeferred(true) + return newProj, transform.NewTree, nil +} + func flattenDistinct(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.Scope, sel RuleSelector, qFlags *sql.QueryFlags) (sql.Node, transform.TreeIdentity, error) { return transform.Node(n, func(n sql.Node) (sql.Node, transform.TreeIdentity, error) { if d, ok := n.(*plan.Distinct); ok { diff --git a/sql/analyzer/rule_ids.go b/sql/analyzer/rule_ids.go index 8aa216c062..3078f2c8e0 100644 --- a/sql/analyzer/rule_ids.go +++ b/sql/analyzer/rule_ids.go @@ -16,45 +16,45 @@ const ( resolveNamedWindowsId // resolveNamedWindows resolveSetVariablesId // resolveSetVariables resolveViewsId // resolveViews - liftCtesId // liftCtes - resolveCtesId // resolveCtes - liftRecursiveCtesId // liftRecursiveCtes - resolveDatabasesId // resolveDatabases - resolveTablesId // resolveTables - loadStoredProceduresId // loadStoredProcedures - validateDropTablesId // validateDropTables - pruneDropTablesId // pruneDropTables - setTargetSchemasId // setTargetSchemas - resolveCreateLikeId // resolveCreateLike - parseColumnDefaultsId // parseColumnDefaults - resolveDropConstraintId // resolveDropConstraint - validateDropConstraintId // validateDropConstraint - loadCheckConstraintsId // loadCheckConstraints - assignCatalogId // assignCatalog - resolveAnalyzeTablesId // resolveAnalyzeTables - resolveCreateSelectId // resolveCreateSelect - resolveSubqueriesId // resolveSubqueries - setViewTargetSchemaId // setViewTargetSchema - resolveUnionsId // resolveUnions - resolveDescribeQueryId // resolveDescribeQuery - checkUniqueTableNamesId // checkUniqueTableNames - resolveTableFunctionsId // resolveTableFunctions - resolveDeclarationsId // resolveDeclarations - resolveColumnDefaultsId // resolveColumnDefaults - ValidateColumnDefaultsId // validateColumnDefaults - validateCreateTriggerId // validateCreateTrigger - validateCreateProcedureId // validateCreateProcedure - resolveCreateProcedureId // resolveCreateProcedure - loadInfoSchemaId // loadInfoSchema - validateReadOnlyDatabaseId // validateReadOnlyDatabase - validateReadOnlyTransactionId // validateReadOnlyTransaction - validateDatabaseSetId // validateDatabaseSet - validatePrivilegesId // validatePrivileges - reresolveTablesId // reresolveTables - setInsertColumnsId // setInsertColumns - validateJoinComplexityId // validateJoinComplexity - applyBinlogReplicaControllerId // applyBinlogReplicaController - applyEventSchedulerId // applyEventScheduler + liftCtesId // liftCtes + resolveCtesId // resolveCtes + liftRecursiveCtesId // liftRecursiveCtes + resolveDatabasesId // resolveDatabases + resolveTablesId // resolveTables + loadStoredProceduresId // loadStoredProcedures + validateDropTablesId // validateDropTables + pruneDropTablesId // pruneDropTables + setTargetSchemasId // setTargetSchemas + resolveCreateLikeId // resolveCreateLike + parseColumnDefaultsId // parseColumnDefaults + resolveDropConstraintId // resolveDropConstraint + validateDropConstraintId // validateDropConstraint + loadCheckConstraintsId // loadCheckConstraints + assignCatalogId // assignCatalog + resolveAnalyzeTablesId // resolveAnalyzeTables + resolveCreateSelectId // resolveCreateSelect + resolveSubqueriesId // resolveSubqueries + setViewTargetSchemaId // setViewTargetSchema + resolveUnionsId // resolveUnions + resolveDescribeQueryId // resolveDescribeQuery + checkUniqueTableNamesId // checkUniqueTableNames + resolveTableFunctionsId // resolveTableFunctions + resolveDeclarationsId // resolveDeclarations + resolveColumnDefaultsId // resolveColumnDefaults + ValidateColumnDefaultsId // validateColumnDefaults + validateCreateTriggerId // validateCreateTrigger + validateCreateProcedureId // validateCreateProcedure + resolveCreateProcedureId // resolveCreateProcedure + loadInfoSchemaId // loadInfoSchema + validateReadOnlyDatabaseId // validateReadOnlyDatabase + validateReadOnlyTransactionId // validateReadOnlyTransaction + validateDatabaseSetId // validateDatabaseSet + validatePrivilegesId // validatePrivileges + reresolveTablesId // reresolveTables + setInsertColumnsId // setInsertColumns + validateJoinComplexityId // validateJoinComplexity + applyBinlogReplicaControllerId // applyBinlogReplicaController + applyEventSchedulerId // applyEventScheduler // default resolveUsingJoinsId // resolveUsingJoins diff --git a/sql/analyzer/rules.go b/sql/analyzer/rules.go index 5d3f3dc3e4..2826bf6f28 100644 --- a/sql/analyzer/rules.go +++ b/sql/analyzer/rules.go @@ -40,7 +40,7 @@ var OnceBeforeDefault = []Rule{ {applyDefaultSelectLimitId, applyDefaultSelectLimit}, {replaceCountStarId, replaceCountStar}, {applyEventSchedulerId, applyEventScheduler}, - {validateOffsetAndLimitId, validateLimitAndOffset}, + {validateOffsetAndLimitId, validateOffsetAndLimit}, {validateCreateTableId, validateCreateTable}, {validateAlterTableId, validateAlterTable}, {validateExprSemId, validateExprSem}, diff --git a/sql/analyzer/validation_rules.go b/sql/analyzer/validation_rules.go index 6b0f792d16..54cf51637d 100644 --- a/sql/analyzer/validation_rules.go +++ b/sql/analyzer/validation_rules.go @@ -29,8 +29,8 @@ import ( "github.com/dolthub/go-mysql-server/sql/types" ) -// validateLimitAndOffset ensures that only integer literals are used for limit and offset values -func validateLimitAndOffset(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.Scope, sel RuleSelector, qFlags *sql.QueryFlags) (sql.Node, transform.TreeIdentity, error) { +// validateOffsetAndLimit ensures that only integer literals are used for limit and offset values +func validateOffsetAndLimit(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.Scope, sel RuleSelector, qFlags *sql.QueryFlags) (sql.Node, transform.TreeIdentity, error) { var err error var i, i64 interface{} transform.Inspect(n, func(n sql.Node) bool { diff --git a/sql/analyzer/warnings.go b/sql/analyzer/warnings.go index 58e741ba37..87807eb377 100644 --- a/sql/analyzer/warnings.go +++ b/sql/analyzer/warnings.go @@ -16,8 +16,7 @@ package analyzer import ( "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/expression" -"github.com/dolthub/go-mysql-server/sql/plan" + "github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/transform" ) @@ -41,23 +40,3 @@ func clearWarnings(ctx *sql.Context, a *Analyzer, node sql.Node, scope *plan.Sco ctx.ClearWarnings() return node, transform.SameTree, nil } - -// TODO: move this somewhere else -func deferProjections(ctx *sql.Context, a *Analyzer, node sql.Node, scope *plan.Scope, sel RuleSelector, qFlags *sql.QueryFlags) (sql.Node, transform.TreeIdentity, error) { - if !a.ServerMode { - return node, transform.SameTree, nil - } - // Find top-level projection, and mark as deferred - if proj, ok := node.(*plan.Project); ok { - // Default value expressions require a second pass, so punt on deferring for now - for _, expr := range proj.Projections { - switch expr.(type) { - case *expression.Wrapper, *sql.ColumnDefaultValue: - return node, transform.SameTree, nil - } - } - proj.Deferred = true - return proj, transform.NewTree, nil - } - return node, transform.SameTree, nil -} \ No newline at end of file diff --git a/sql/plan/process.go b/sql/plan/process.go index b154c03e40..f74d4d1b47 100644 --- a/sql/plan/process.go +++ b/sql/plan/process.go @@ -307,7 +307,7 @@ const ( ) type TrackedRowIter struct { - Node sql.Node + node sql.Node iter sql.RowIter numRows int64 QueryType queryType @@ -322,7 +322,7 @@ func NewTrackedRowIter( onNext NotifyFunc, onDone NotifyFunc, ) *TrackedRowIter { - return &TrackedRowIter{Node: node, iter: iter, onDone: onDone, onNext: onNext} + return &TrackedRowIter{node: node, iter: iter, onDone: onDone, onNext: onNext} } func (i *TrackedRowIter) done() { @@ -330,9 +330,9 @@ func (i *TrackedRowIter) done() { i.onDone() i.onDone = nil } - if i.Node != nil { + if i.node != nil { i.Dispose() - i.Node = nil + i.node = nil } } @@ -348,8 +348,8 @@ func disposeNode(n sql.Node) { } func (i *TrackedRowIter) Dispose() { - if i.Node != nil { - disposeNode(i.Node) + if i.node != nil { + disposeNode(i.node) } } @@ -377,6 +377,10 @@ func (i *TrackedRowIter) Close(ctx *sql.Context) error { return err } +func (i *TrackedRowIter) GetNode() sql.Node { + return i.node +} + func (i *TrackedRowIter) updateSessionVars(ctx *sql.Context) { switch i.QueryType { case QueryTypeSelect: diff --git a/sql/plan/project.go b/sql/plan/project.go index 794db66f86..617ceb4670 100644 --- a/sql/plan/project.go +++ b/sql/plan/project.go @@ -26,7 +26,6 @@ import ( // Project is a projection of certain expression from the children node. type Project struct { UnaryNode - // Expression projected. Projections []sql.Expression Deferred bool } @@ -161,10 +160,9 @@ func (p *Project) WithChildren(children ...sql.Node) (sql.Node, error) { if len(children) != 1 { return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 1) } - - np := NewProject(p.Projections, children[0]) - np.Deferred = p.Deferred - return np, nil + np := *p + np.Child = children[0] + return &np, nil } // CheckPrivileges implements the interface sql.Node. @@ -182,8 +180,13 @@ func (p *Project) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { if len(exprs) != len(p.Projections) { return nil, sql.ErrInvalidChildrenNumber.New(p, len(exprs), len(p.Projections)) } + np := *p + np.Projections = exprs + return &np, nil +} - np := NewProject(exprs, p.Child) - np.Deferred = p.Deferred - return np, nil +func (p *Project) WithDeferred(deferred bool) *Project { + np := *p + np.Deferred = deferred + return &np } From 9846d13b5ef9024c182f326245bdf475f792deb8 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 25 Sep 2024 16:05:41 -0700 Subject: [PATCH 10/39] fix --- enginetest/enginetests.go | 11 +---------- server/handler.go | 10 +++++----- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/enginetest/enginetests.go b/enginetest/enginetests.go index ae6680f8de..2ec5a10304 100644 --- a/enginetest/enginetests.go +++ b/enginetest/enginetests.go @@ -5636,16 +5636,7 @@ func TestTypesOverWire(t *testing.T, harness ClientHarness, sessionBuilder serve require.NoError(t, err) expectedRowSet := script.Results[queryIdx] expectedRowIdx := 0 - var projs []sql.Expression - if trackedIter, ok := engineIter.(*plan.TrackedRowIter); ok { - if commitNode, ok := trackedIter.Node.(*plan.TransactionCommittingNode); ok { - if proj, ok := commitNode.Child().(*plan.Project); ok { - if proj.Deferred { - projs = proj.Projections - } - } - } - } + projs := server.GetDeferredProjections(engineIter) var engineRow sql.Row for engineRow, err = engineIter.Next(ctx); err == nil; engineRow, err = engineIter.Next(ctx) { if !assert.True(t, r.Next()) { diff --git a/server/handler.go b/server/handler.go index f36357c97f..bb2c4c63e7 100644 --- a/server/handler.go +++ b/server/handler.go @@ -512,11 +512,11 @@ func resultForEmptyIter(ctx *sql.Context, iter sql.RowIter, resultFields []*quer return &sqltypes.Result{Fields: resultFields}, nil } -// getDeferredProjections looks for a top-level deferred projection -func getDeferredProjections(iter sql.RowIter) []sql.Expression { +// GetDeferredProjections looks for a top-level deferred projection +func GetDeferredProjections(iter sql.RowIter) []sql.Expression { switch i := iter.(type) { case *rowexec.ExprCloserIter: - return getDeferredProjections(i.Iter) + return GetDeferredProjections(i.Iter) case *plan.TrackedRowIter: if commit, isCommit := i.GetNode().(*plan.TransactionCommittingNode); isCommit { if proj, isProj := commit.Child().(*plan.Project); isProj { @@ -546,7 +546,7 @@ func resultForMax1RowIter(ctx *sql.Context, schema sql.Schema, iter sql.RowIter, return nil, err } - projs := getDeferredProjections(iter) + projs := GetDeferredProjections(iter) outputRow, err := RowToSQL(ctx, schema, row, projs) if err != nil { return nil, err @@ -624,7 +624,7 @@ func (h *Handler) resultForDefaultIter( timer := time.NewTimer(waitTime) defer timer.Stop() - projs := getDeferredProjections(iter) + projs := GetDeferredProjections(iter) // Reads rows from the channel, converts them to wire format, // and calls |callback| to give them to vitess. eg.Go(func() error { From 6539989d790e0a3fbe0bf6d6ba14025507c2b0c2 Mon Sep 17 00:00:00 2001 From: jycor Date: Wed, 25 Sep 2024 23:07:23 +0000 Subject: [PATCH 11/39] [ga-format-pr] Run ./format_repo.sh to fix formatting --- enginetest/memory_engine_test.go | 2 +- sql/analyzer/rule_ids.go | 80 ++++++++++++++++---------------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/enginetest/memory_engine_test.go b/enginetest/memory_engine_test.go index f88a6e09e2..345e32cf5e 100644 --- a/enginetest/memory_engine_test.go +++ b/enginetest/memory_engine_test.go @@ -210,7 +210,7 @@ func TestSingleScript(t *testing.T) { { Name: "test script", SetUpScript: []string{}, - Assertions: []queries.ScriptTestAssertion{ + Assertions: []queries.ScriptTestAssertion{ { Query: "SELECT REGEXP_LIKE('testing', 'TESTING');", Expected: []sql.Row{ diff --git a/sql/analyzer/rule_ids.go b/sql/analyzer/rule_ids.go index 3078f2c8e0..74911d2ad8 100644 --- a/sql/analyzer/rule_ids.go +++ b/sql/analyzer/rule_ids.go @@ -16,45 +16,45 @@ const ( resolveNamedWindowsId // resolveNamedWindows resolveSetVariablesId // resolveSetVariables resolveViewsId // resolveViews - liftCtesId // liftCtes - resolveCtesId // resolveCtes - liftRecursiveCtesId // liftRecursiveCtes - resolveDatabasesId // resolveDatabases - resolveTablesId // resolveTables - loadStoredProceduresId // loadStoredProcedures - validateDropTablesId // validateDropTables - pruneDropTablesId // pruneDropTables - setTargetSchemasId // setTargetSchemas - resolveCreateLikeId // resolveCreateLike - parseColumnDefaultsId // parseColumnDefaults - resolveDropConstraintId // resolveDropConstraint - validateDropConstraintId // validateDropConstraint - loadCheckConstraintsId // loadCheckConstraints - assignCatalogId // assignCatalog - resolveAnalyzeTablesId // resolveAnalyzeTables - resolveCreateSelectId // resolveCreateSelect - resolveSubqueriesId // resolveSubqueries - setViewTargetSchemaId // setViewTargetSchema - resolveUnionsId // resolveUnions - resolveDescribeQueryId // resolveDescribeQuery - checkUniqueTableNamesId // checkUniqueTableNames - resolveTableFunctionsId // resolveTableFunctions - resolveDeclarationsId // resolveDeclarations - resolveColumnDefaultsId // resolveColumnDefaults - ValidateColumnDefaultsId // validateColumnDefaults - validateCreateTriggerId // validateCreateTrigger - validateCreateProcedureId // validateCreateProcedure - resolveCreateProcedureId // resolveCreateProcedure - loadInfoSchemaId // loadInfoSchema - validateReadOnlyDatabaseId // validateReadOnlyDatabase - validateReadOnlyTransactionId // validateReadOnlyTransaction - validateDatabaseSetId // validateDatabaseSet - validatePrivilegesId // validatePrivileges - reresolveTablesId // reresolveTables - setInsertColumnsId // setInsertColumns - validateJoinComplexityId // validateJoinComplexity - applyBinlogReplicaControllerId // applyBinlogReplicaController - applyEventSchedulerId // applyEventScheduler + liftCtesId // liftCtes + resolveCtesId // resolveCtes + liftRecursiveCtesId // liftRecursiveCtes + resolveDatabasesId // resolveDatabases + resolveTablesId // resolveTables + loadStoredProceduresId // loadStoredProcedures + validateDropTablesId // validateDropTables + pruneDropTablesId // pruneDropTables + setTargetSchemasId // setTargetSchemas + resolveCreateLikeId // resolveCreateLike + parseColumnDefaultsId // parseColumnDefaults + resolveDropConstraintId // resolveDropConstraint + validateDropConstraintId // validateDropConstraint + loadCheckConstraintsId // loadCheckConstraints + assignCatalogId // assignCatalog + resolveAnalyzeTablesId // resolveAnalyzeTables + resolveCreateSelectId // resolveCreateSelect + resolveSubqueriesId // resolveSubqueries + setViewTargetSchemaId // setViewTargetSchema + resolveUnionsId // resolveUnions + resolveDescribeQueryId // resolveDescribeQuery + checkUniqueTableNamesId // checkUniqueTableNames + resolveTableFunctionsId // resolveTableFunctions + resolveDeclarationsId // resolveDeclarations + resolveColumnDefaultsId // resolveColumnDefaults + ValidateColumnDefaultsId // validateColumnDefaults + validateCreateTriggerId // validateCreateTrigger + validateCreateProcedureId // validateCreateProcedure + resolveCreateProcedureId // resolveCreateProcedure + loadInfoSchemaId // loadInfoSchema + validateReadOnlyDatabaseId // validateReadOnlyDatabase + validateReadOnlyTransactionId // validateReadOnlyTransaction + validateDatabaseSetId // validateDatabaseSet + validatePrivilegesId // validatePrivileges + reresolveTablesId // reresolveTables + setInsertColumnsId // setInsertColumns + validateJoinComplexityId // validateJoinComplexity + applyBinlogReplicaControllerId // applyBinlogReplicaController + applyEventSchedulerId // applyEventScheduler // default resolveUsingJoinsId // resolveUsingJoins @@ -141,7 +141,7 @@ const ( cacheSubqueryResultsId // cacheSubqueryResults cacheSubqueryAliasesInJoinsId // cacheSubqueryAliasesInJoins backtickDefaulColumnValueNamesId // backtickDefaulColumnValueNames - DeferProjectionsId // deferProjections + DeferProjectionsId // deferProjections AutocommitId // addAutocommitNode TrackProcessId // trackProcess parallelizeId // parallelize From 4804e5364b0d46998e8457b8fb0175cd49f1e5fa Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 25 Sep 2024 16:14:32 -0700 Subject: [PATCH 12/39] more tidying --- enginetest/memory_engine_test.go | 12 ++---------- server/handler.go | 2 +- sql/rowexec/dml.go | 2 +- sql/rowexec/expr_closer.go | 12 ++++++++---- sql/rowexec/rel.go | 8 ++++---- sql/rowexec/rel_iters.go | 10 +++++----- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/enginetest/memory_engine_test.go b/enginetest/memory_engine_test.go index f88a6e09e2..204296e865 100644 --- a/enginetest/memory_engine_test.go +++ b/enginetest/memory_engine_test.go @@ -205,25 +205,17 @@ func newUpdateResult(matched, updated int) types.OkResult { // Convenience test for debugging a single query. Unskip and set to the desired query. func TestSingleScript(t *testing.T) { - //t.Skip() + t.Skip() var scripts = []queries.ScriptTest{ { Name: "test script", SetUpScript: []string{}, - Assertions: []queries.ScriptTestAssertion{ - { - Query: "SELECT REGEXP_LIKE('testing', 'TESTING');", - Expected: []sql.Row{ - {0}, - }, - }, - }, + Assertions: []queries.ScriptTestAssertion{}, }, } for _, test := range scripts { harness := enginetest.NewMemoryHarness("", 1, testNumPartitions, true, nil) - harness.UseServer() engine, err := harness.NewEngine(t) if err != nil { panic(err) diff --git a/server/handler.go b/server/handler.go index bb2c4c63e7..85869aff72 100644 --- a/server/handler.go +++ b/server/handler.go @@ -516,7 +516,7 @@ func resultForEmptyIter(ctx *sql.Context, iter sql.RowIter, resultFields []*quer func GetDeferredProjections(iter sql.RowIter) []sql.Expression { switch i := iter.(type) { case *rowexec.ExprCloserIter: - return GetDeferredProjections(i.Iter) + return GetDeferredProjections(i.GetIter()) case *plan.TrackedRowIter: if commit, isCommit := i.GetNode().(*plan.TransactionCommittingNode); isCommit { if proj, isProj := commit.Child().(*plan.Project); isProj { diff --git a/sql/rowexec/dml.go b/sql/rowexec/dml.go index 4c86bf17b5..b10c9c4044 100644 --- a/sql/rowexec/dml.go +++ b/sql/rowexec/dml.go @@ -375,7 +375,7 @@ func (b *BaseBuilder) buildRowUpdateAccumulator(ctx *sql.Context, n *plan.RowUpd case *updateJoinIter: i.accumulator = rowHandler.(*updateJoinRowHandler) done = true - case *ProjectIter: + case *projectIter: iter = i.childIter case *plan.CheckpointingTableEditorIter: iter = i.InnerIter() diff --git a/sql/rowexec/expr_closer.go b/sql/rowexec/expr_closer.go index 1a8be52770..0b4e43adb0 100644 --- a/sql/rowexec/expr_closer.go +++ b/sql/rowexec/expr_closer.go @@ -22,7 +22,7 @@ import ( // ExprCloserIter ensures that all expressions that implement sql.Closer are closed. This is implemented as a capturing // iterator, as our workflow only supports closing nodes, not expressions. type ExprCloserIter struct { - Iter sql.RowIter + iter sql.RowIter exprs []sql.Closer } @@ -44,19 +44,19 @@ func AddExpressionCloser(node sql.Node, iter sql.RowIter) sql.RowIter { return iter } return &ExprCloserIter{ - Iter: iter, + iter: iter, exprs: exprs, } } // Next implements the interface sql.RowIter. func (eci *ExprCloserIter) Next(ctx *sql.Context) (sql.Row, error) { - return eci.Iter.Next(ctx) + return eci.iter.Next(ctx) } // Close implements the interface sql.RowIter. func (eci *ExprCloserIter) Close(ctx *sql.Context) error { - err := eci.Iter.Close(ctx) + err := eci.iter.Close(ctx) for _, expr := range eci.exprs { if nErr := expr.Close(ctx); err == nil { err = nErr @@ -64,3 +64,7 @@ func (eci *ExprCloserIter) Close(ctx *sql.Context) error { } return err } + +func (eci *ExprCloserIter) GetIter() sql.RowIter { + return eci.iter +} diff --git a/sql/rowexec/rel.go b/sql/rowexec/rel.go index 510b03ed68..8344d68026 100644 --- a/sql/rowexec/rel.go +++ b/sql/rowexec/rel.go @@ -310,8 +310,8 @@ func (b *BaseBuilder) buildProject(ctx *sql.Context, n *plan.Project, row sql.Ro return nil, err } - return sql.NewSpanIter(span, &ProjectIter{ - Projs: n.Projections, + return sql.NewSpanIter(span, &projectIter{ + projs: n.Projections, deferred: n.Deferred, childIter: i, }), nil @@ -322,8 +322,8 @@ func (b *BaseBuilder) buildVirtualColumnTable(ctx *sql.Context, n *plan.VirtualC attribute.Int("projections", len(n.Projections)), )) - return sql.NewSpanIter(span, &ProjectIter{ - Projs: n.Projections, + return sql.NewSpanIter(span, &projectIter{ + projs: n.Projections, childIter: tableIter, }), nil } diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index 0b9e66de9a..abb1c1b73b 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -503,13 +503,13 @@ func (di *orderedDistinctIter) Close(ctx *sql.Context) error { return di.childIter.Close(ctx) } -type ProjectIter struct { - Projs []sql.Expression +type projectIter struct { + projs []sql.Expression deferred bool childIter sql.RowIter } -func (i *ProjectIter) Next(ctx *sql.Context) (sql.Row, error) { +func (i *projectIter) Next(ctx *sql.Context) (sql.Row, error) { childRow, err := i.childIter.Next(ctx) if err != nil { return nil, err @@ -517,10 +517,10 @@ func (i *ProjectIter) Next(ctx *sql.Context) (sql.Row, error) { if i.deferred { return childRow, nil } - return ProjectRow(ctx, i.Projs, childRow) + return ProjectRow(ctx, i.projs, childRow) } -func (i *ProjectIter) Close(ctx *sql.Context) error { +func (i *projectIter) Close(ctx *sql.Context) error { return i.childIter.Close(ctx) } From 94ff580a6d11873b08048f1947395a25235d419e Mon Sep 17 00:00:00 2001 From: jycor Date: Wed, 25 Sep 2024 23:18:15 +0000 Subject: [PATCH 13/39] [ga-format-pr] Run ./format_repo.sh to fix formatting --- enginetest/memory_engine_test.go | 2 +- sql/analyzer/rule_ids.go | 78 ++++++++++++++++---------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/enginetest/memory_engine_test.go b/enginetest/memory_engine_test.go index 06ee059340..204296e865 100644 --- a/enginetest/memory_engine_test.go +++ b/enginetest/memory_engine_test.go @@ -210,7 +210,7 @@ func TestSingleScript(t *testing.T) { { Name: "test script", SetUpScript: []string{}, - Assertions: []queries.ScriptTestAssertion{}, + Assertions: []queries.ScriptTestAssertion{}, }, } diff --git a/sql/analyzer/rule_ids.go b/sql/analyzer/rule_ids.go index 0116086ff3..4936e23958 100644 --- a/sql/analyzer/rule_ids.go +++ b/sql/analyzer/rule_ids.go @@ -16,45 +16,45 @@ const ( resolveNamedWindowsId // resolveNamedWindows resolveSetVariablesId // resolveSetVariables resolveViewsId // resolveViews - liftCtesId // liftCtes - resolveCtesId // resolveCtes - liftRecursiveCtesId // liftRecursiveCtes - resolveDatabasesId // resolveDatabases - resolveTablesId // resolveTables - loadStoredProceduresId // loadStoredProcedures - validateDropTablesId // validateDropTables - pruneDropTablesId // pruneDropTables - setTargetSchemasId // setTargetSchemas - resolveCreateLikeId // resolveCreateLike - parseColumnDefaultsId // parseColumnDefaults - resolveDropConstraintId // resolveDropConstraint - validateDropConstraintId // validateDropConstraint - loadCheckConstraintsId // loadCheckConstraints - assignCatalogId // assignCatalog - resolveAnalyzeTablesId // resolveAnalyzeTables - resolveCreateSelectId // resolveCreateSelect - resolveSubqueriesId // resolveSubqueries - setViewTargetSchemaId // setViewTargetSchema - resolveUnionsId // resolveUnions - resolveDescribeQueryId // resolveDescribeQuery - checkUniqueTableNamesId // checkUniqueTableNames - resolveTableFunctionsId // resolveTableFunctions - resolveDeclarationsId // resolveDeclarations - resolveColumnDefaultsId // resolveColumnDefaults - ValidateColumnDefaultsId // validateColumnDefaults - validateCreateTriggerId // validateCreateTrigger - validateCreateProcedureId // validateCreateProcedure - resolveCreateProcedureId // resolveCreateProcedure - loadInfoSchemaId // loadInfoSchema - validateReadOnlyDatabaseId // validateReadOnlyDatabase - validateReadOnlyTransactionId // validateReadOnlyTransaction - validateDatabaseSetId // validateDatabaseSet - validatePrivilegesId // validatePrivileges - reresolveTablesId // reresolveTables - setInsertColumnsId // setInsertColumns - validateJoinComplexityId // validateJoinComplexity - applyBinlogReplicaControllerId // applyBinlogReplicaController - applyEventSchedulerId // applyEventScheduler + liftCtesId // liftCtes + resolveCtesId // resolveCtes + liftRecursiveCtesId // liftRecursiveCtes + resolveDatabasesId // resolveDatabases + resolveTablesId // resolveTables + loadStoredProceduresId // loadStoredProcedures + validateDropTablesId // validateDropTables + pruneDropTablesId // pruneDropTables + setTargetSchemasId // setTargetSchemas + resolveCreateLikeId // resolveCreateLike + parseColumnDefaultsId // parseColumnDefaults + resolveDropConstraintId // resolveDropConstraint + validateDropConstraintId // validateDropConstraint + loadCheckConstraintsId // loadCheckConstraints + assignCatalogId // assignCatalog + resolveAnalyzeTablesId // resolveAnalyzeTables + resolveCreateSelectId // resolveCreateSelect + resolveSubqueriesId // resolveSubqueries + setViewTargetSchemaId // setViewTargetSchema + resolveUnionsId // resolveUnions + resolveDescribeQueryId // resolveDescribeQuery + checkUniqueTableNamesId // checkUniqueTableNames + resolveTableFunctionsId // resolveTableFunctions + resolveDeclarationsId // resolveDeclarations + resolveColumnDefaultsId // resolveColumnDefaults + ValidateColumnDefaultsId // validateColumnDefaults + validateCreateTriggerId // validateCreateTrigger + validateCreateProcedureId // validateCreateProcedure + resolveCreateProcedureId // resolveCreateProcedure + loadInfoSchemaId // loadInfoSchema + validateReadOnlyDatabaseId // validateReadOnlyDatabase + validateReadOnlyTransactionId // validateReadOnlyTransaction + validateDatabaseSetId // validateDatabaseSet + validatePrivilegesId // validatePrivileges + reresolveTablesId // reresolveTables + setInsertColumnsId // setInsertColumns + validateJoinComplexityId // validateJoinComplexity + applyBinlogReplicaControllerId // applyBinlogReplicaController + applyEventSchedulerId // applyEventScheduler // default resolveUsingJoinsId // resolveUsingJoins From a35cb43a8ff10211bb7b5c8638bcfcb87214aca7 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 30 Sep 2024 11:47:27 -0700 Subject: [PATCH 14/39] conditionally defer projections --- server/handler.go | 11 ++++++----- sql/analyzer/optimization_rules.go | 7 ++++--- sql/plan/process.go | 4 ++++ sql/plan/project.go | 6 +++--- sql/rowexec/dml.go | 2 +- sql/rowexec/other_iters.go | 4 ++-- sql/rowexec/rel.go | 6 +++--- sql/rowexec/rel_iters.go | 19 ++++++++++++++++--- sql/rowexec/transaction.go | 2 +- sql/rowexec/transaction_iters.go | 12 ++++++++---- sql/rowexec/update.go | 2 +- 11 files changed, 49 insertions(+), 26 deletions(-) diff --git a/server/handler.go b/server/handler.go index 85869aff72..dd7d066a7b 100644 --- a/server/handler.go +++ b/server/handler.go @@ -512,16 +512,17 @@ func resultForEmptyIter(ctx *sql.Context, iter sql.RowIter, resultFields []*quer return &sqltypes.Result{Fields: resultFields}, nil } -// GetDeferredProjections looks for a top-level deferred projection +// GetDeferredProjections looks for a top-level deferred projection, marks it as deferred, and retrieves its projections func GetDeferredProjections(iter sql.RowIter) []sql.Expression { switch i := iter.(type) { case *rowexec.ExprCloserIter: return GetDeferredProjections(i.GetIter()) case *plan.TrackedRowIter: - if commit, isCommit := i.GetNode().(*plan.TransactionCommittingNode); isCommit { - if proj, isProj := commit.Child().(*plan.Project); isProj { - if proj.Deferred { - return proj.Projections + if commitIter, isCommitIter := i.GetIter().(*rowexec.TransactionCommittingIter); isCommitIter { + if projIter, isProjIter := commitIter.GetIter().(*rowexec.ProjectIter); isProjIter { + if projIter.CanDefer() { + projIter.Defer() + return projIter.GetProjections() } } } diff --git a/sql/analyzer/optimization_rules.go b/sql/analyzer/optimization_rules.go index bc9f515793..4f2ff868c0 100644 --- a/sql/analyzer/optimization_rules.go +++ b/sql/analyzer/optimization_rules.go @@ -54,8 +54,9 @@ func eraseProjection(ctx *sql.Context, a *Analyzer, node sql.Node, scope *plan.S }) } -// deferProjections defers projections to the end of the query execution. This can avoid an unnecessary slice allocation -// specifically in the case where we are spooling rows from the server to the client. +// deferProjections marks projection nodes that can defer projections to the end of the query execution. +// This can avoid an unnecessary slice allocation specifically in the case where we are spooling rows +// from the server to the client. func deferProjections(ctx *sql.Context, a *Analyzer, node sql.Node, scope *plan.Scope, sel RuleSelector, qFlags *sql.QueryFlags) (sql.Node, transform.TreeIdentity, error) { if !a.ServerMode { return node, transform.SameTree, nil @@ -72,7 +73,7 @@ func deferProjections(ctx *sql.Context, a *Analyzer, node sql.Node, scope *plan. return node, transform.SameTree, nil } } - newProj := proj.WithDeferred(true) + newProj := proj.WithCanDefer(true) return newProj, transform.NewTree, nil } diff --git a/sql/plan/process.go b/sql/plan/process.go index f74d4d1b47..07b62b0fbb 100644 --- a/sql/plan/process.go +++ b/sql/plan/process.go @@ -381,6 +381,10 @@ func (i *TrackedRowIter) GetNode() sql.Node { return i.node } +func (i *TrackedRowIter) GetIter() sql.RowIter { + return i.iter +} + func (i *TrackedRowIter) updateSessionVars(ctx *sql.Context) { switch i.QueryType { case QueryTypeSelect: diff --git a/sql/plan/project.go b/sql/plan/project.go index 617ceb4670..814f63bd13 100644 --- a/sql/plan/project.go +++ b/sql/plan/project.go @@ -27,7 +27,7 @@ import ( type Project struct { UnaryNode Projections []sql.Expression - Deferred bool + CanDefer bool } var _ sql.Expressioner = (*Project)(nil) @@ -185,8 +185,8 @@ func (p *Project) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { return &np, nil } -func (p *Project) WithDeferred(deferred bool) *Project { +func (p *Project) WithCanDefer(canDefer bool) *Project { np := *p - np.Deferred = deferred + np.CanDefer = canDefer return &np } diff --git a/sql/rowexec/dml.go b/sql/rowexec/dml.go index b10c9c4044..4c86bf17b5 100644 --- a/sql/rowexec/dml.go +++ b/sql/rowexec/dml.go @@ -375,7 +375,7 @@ func (b *BaseBuilder) buildRowUpdateAccumulator(ctx *sql.Context, n *plan.RowUpd case *updateJoinIter: i.accumulator = rowHandler.(*updateJoinRowHandler) done = true - case *projectIter: + case *ProjectIter: iter = i.childIter case *plan.CheckpointingTableEditorIter: iter = i.InnerIter() diff --git a/sql/rowexec/other_iters.go b/sql/rowexec/other_iters.go index 09f0b5961b..dcc2e9b713 100644 --- a/sql/rowexec/other_iters.go +++ b/sql/rowexec/other_iters.go @@ -311,8 +311,8 @@ type rowIterPartitionFunc func(ctx *sql.Context, partition sql.Partition) (sql.R // iterPartitionRows is the parallel worker for an Exchange node. It // is meant to be run as a goroutine in an errgroup.Group. It will // values read off of |partitions|. For each value it reads, it will -// call |getRowIter| to get a row projectIter, and will then call |Next| on -// that row projectIter, passing every row it gets into |rows|. If it +// call |getRowIter| to get a row ProjectIter, and will then call |Next| on +// that row ProjectIter, passing every row it gets into |rows|. If it // receives an error at any point, it returns it. |iterPartitionRows| // stops iterating and returns |nil| when |partitions| is closed. func iterPartitionRows(ctx *sql.Context, getRowIter rowIterPartitionFunc, partitions <-chan sql.Partition, rows chan<- sql.Row) (rerr error) { diff --git a/sql/rowexec/rel.go b/sql/rowexec/rel.go index a40093ef61..17bb1c7f37 100644 --- a/sql/rowexec/rel.go +++ b/sql/rowexec/rel.go @@ -311,9 +311,9 @@ func (b *BaseBuilder) buildProject(ctx *sql.Context, n *plan.Project, row sql.Ro return nil, err } - return sql.NewSpanIter(span, &projectIter{ + return sql.NewSpanIter(span, &ProjectIter{ projs: n.Projections, - deferred: n.Deferred, + canDefer: n.CanDefer, childIter: i, }), nil } @@ -323,7 +323,7 @@ func (b *BaseBuilder) buildVirtualColumnTable(ctx *sql.Context, n *plan.VirtualC attribute.Int("projections", len(n.Projections)), )) - return sql.NewSpanIter(span, &projectIter{ + return sql.NewSpanIter(span, &ProjectIter{ projs: n.Projections, childIter: tableIter, }), nil diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index bd327e6426..0131b4e0cb 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -124,13 +124,14 @@ func (i *offsetIter) Close(ctx *sql.Context) error { var _ sql.RowIter = &iters.JsonTableRowIter{} -type projectIter struct { +type ProjectIter struct { projs []sql.Expression + canDefer bool deferred bool childIter sql.RowIter } -func (i *projectIter) Next(ctx *sql.Context) (sql.Row, error) { +func (i *ProjectIter) Next(ctx *sql.Context) (sql.Row, error) { childRow, err := i.childIter.Next(ctx) if err != nil { return nil, err @@ -141,10 +142,22 @@ func (i *projectIter) Next(ctx *sql.Context) (sql.Row, error) { return ProjectRow(ctx, i.projs, childRow) } -func (i *projectIter) Close(ctx *sql.Context) error { +func (i *ProjectIter) Close(ctx *sql.Context) error { return i.childIter.Close(ctx) } +func (i *ProjectIter) GetProjections() []sql.Expression { + return i.projs +} + +func (i *ProjectIter) CanDefer() bool { + return i.canDefer +} + +func (i *ProjectIter) Defer() { + i.deferred = true +} + // ProjectRow evaluates a set of projections. func ProjectRow( ctx *sql.Context, diff --git a/sql/rowexec/transaction.go b/sql/rowexec/transaction.go index ecfd3a5017..c036712d31 100644 --- a/sql/rowexec/transaction.go +++ b/sql/rowexec/transaction.go @@ -306,5 +306,5 @@ func (b *BaseBuilder) buildTransactionCommittingNode(ctx *sql.Context, n *plan.T if err != nil { return nil, err } - return transactionCommittingIter{childIter: iter}, nil + return &TransactionCommittingIter{childIter: iter}, nil } diff --git a/sql/rowexec/transaction_iters.go b/sql/rowexec/transaction_iters.go index d02cddd34b..f358f96621 100644 --- a/sql/rowexec/transaction_iters.go +++ b/sql/rowexec/transaction_iters.go @@ -67,18 +67,18 @@ func getLockableTable(table sql.Table) (sql.Lockable, error) { } } -// transactionCommittingIter is a simple RowIter wrapper to allow the engine to conditionally commit a transaction +// TransactionCommittingIter is a simple RowIter wrapper to allow the engine to conditionally commit a transaction // during the Close() operation -type transactionCommittingIter struct { +type TransactionCommittingIter struct { childIter sql.RowIter transactionDatabase string } -func (t transactionCommittingIter) Next(ctx *sql.Context) (sql.Row, error) { +func (t *TransactionCommittingIter) Next(ctx *sql.Context) (sql.Row, error) { return t.childIter.Next(ctx) } -func (t transactionCommittingIter) Close(ctx *sql.Context) error { +func (t *TransactionCommittingIter) Close(ctx *sql.Context) error { var err error if t.childIter != nil { err = t.childIter.Close(ctx) @@ -114,3 +114,7 @@ func (t transactionCommittingIter) Close(ctx *sql.Context) error { return nil } + +func (t *TransactionCommittingIter) GetIter() sql.RowIter { + return t.childIter +} diff --git a/sql/rowexec/update.go b/sql/rowexec/update.go index 2fc20b6dff..c61c0d929d 100644 --- a/sql/rowexec/update.go +++ b/sql/rowexec/update.go @@ -183,7 +183,7 @@ func newUpdateIter( } } -// updateJoinIter wraps the child UpdateSource projectIter and returns join row in such a way that updates per table row are +// updateJoinIter wraps the child UpdateSource ProjectIter and returns join row in such a way that updates per table row are // done once. type updateJoinIter struct { updateSourceIter sql.RowIter From a695ea80bae6292915d74f375690221d7e9deea2 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 30 Sep 2024 11:51:07 -0700 Subject: [PATCH 15/39] skip optimization for testing --- enginetest/enginetests.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/enginetest/enginetests.go b/enginetest/enginetests.go index 2ec5a10304..be74c6ecc2 100644 --- a/enginetest/enginetests.go +++ b/enginetest/enginetests.go @@ -5636,7 +5636,6 @@ func TestTypesOverWire(t *testing.T, harness ClientHarness, sessionBuilder serve require.NoError(t, err) expectedRowSet := script.Results[queryIdx] expectedRowIdx := 0 - projs := server.GetDeferredProjections(engineIter) var engineRow sql.Row for engineRow, err = engineIter.Next(ctx); err == nil; engineRow, err = engineIter.Next(ctx) { if !assert.True(t, r.Next()) { @@ -5654,7 +5653,7 @@ func TestTypesOverWire(t *testing.T, harness ClientHarness, sessionBuilder serve break } expectedEngineRow := make([]*string, len(engineRow)) - row, err := server.RowToSQL(ctx, sch, engineRow, projs) + row, err := server.RowToSQL(ctx, sch, engineRow, nil) if !assert.NoError(t, err) { break } From d4239dc983cd00711a0e67acc2a3308e11049b25 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 30 Sep 2024 12:22:11 -0700 Subject: [PATCH 16/39] fix --- server/handler.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/handler.go b/server/handler.go index dd7d066a7b..8afa1eda84 100644 --- a/server/handler.go +++ b/server/handler.go @@ -518,6 +518,11 @@ func GetDeferredProjections(iter sql.RowIter) []sql.Expression { case *rowexec.ExprCloserIter: return GetDeferredProjections(i.GetIter()) case *plan.TrackedRowIter: + // TODO: absolutely no idea why i.GetNode() is nil sometimes and why that means the deferred projections + // don't work but that is how it is + if i.GetNode() == nil { + return nil + } if commitIter, isCommitIter := i.GetIter().(*rowexec.TransactionCommittingIter); isCommitIter { if projIter, isProjIter := commitIter.GetIter().(*rowexec.ProjectIter); isProjIter { if projIter.CanDefer() { @@ -588,6 +593,7 @@ func (h *Handler) resultForDefaultIter( wg.Add(2) // Read rows off the row iterator and send them to the row channel. + projs := GetDeferredProjections(iter) var rowChan = make(chan sql.Row, 512) eg.Go(func() error { defer pan2err() @@ -625,7 +631,6 @@ func (h *Handler) resultForDefaultIter( timer := time.NewTimer(waitTime) defer timer.Stop() - projs := GetDeferredProjections(iter) // Reads rows from the channel, converts them to wire format, // and calls |callback| to give them to vitess. eg.Go(func() error { From c41b9f1776db23431fbad8e907e2a6e3dbddb4e7 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 30 Sep 2024 13:10:40 -0700 Subject: [PATCH 17/39] fix again --- sql/analyzer/optimization_rules.go | 5 +++-- sql/rowexec/rel_iters.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/analyzer/optimization_rules.go b/sql/analyzer/optimization_rules.go index 4f2ff868c0..07d5afdee5 100644 --- a/sql/analyzer/optimization_rules.go +++ b/sql/analyzer/optimization_rules.go @@ -15,7 +15,8 @@ package analyzer import ( - "strings" + "github.com/dolthub/go-mysql-server/sql/expression/function" +"strings" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/expression" @@ -69,7 +70,7 @@ func deferProjections(ctx *sql.Context, a *Analyzer, node sql.Node, scope *plan. // Default value expressions require a second pass, so punt on deferring for now for _, expr := range proj.Projections { switch expr.(type) { - case *expression.Wrapper, *sql.ColumnDefaultValue: + case *expression.Wrapper, *sql.ColumnDefaultValue, function.RowCount: return node, transform.SameTree, nil } } diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index 0131b4e0cb..5bbc7561a6 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -137,7 +137,7 @@ func (i *ProjectIter) Next(ctx *sql.Context) (sql.Row, error) { return nil, err } if i.deferred { - return childRow, nil + //return childRow, nil } return ProjectRow(ctx, i.projs, childRow) } From 87c5caef216078c2512bce18413bbd81a790df0e Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 30 Sep 2024 13:11:38 -0700 Subject: [PATCH 18/39] fix --- sql/analyzer/optimization_rules.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/analyzer/optimization_rules.go b/sql/analyzer/optimization_rules.go index 07d5afdee5..15f9ba63eb 100644 --- a/sql/analyzer/optimization_rules.go +++ b/sql/analyzer/optimization_rules.go @@ -15,11 +15,11 @@ package analyzer import ( - "github.com/dolthub/go-mysql-server/sql/expression/function" -"strings" + "strings" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/expression" + "github.com/dolthub/go-mysql-server/sql/expression/function" "github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/transform" "github.com/dolthub/go-mysql-server/sql/types" From 6bfd2056cecbb5c8b647387ed26025d8364fe9fb Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 30 Sep 2024 13:12:26 -0700 Subject: [PATCH 19/39] aaaaa --- sql/rowexec/rel_iters.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index 5bbc7561a6..0131b4e0cb 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -137,7 +137,7 @@ func (i *ProjectIter) Next(ctx *sql.Context) (sql.Row, error) { return nil, err } if i.deferred { - //return childRow, nil + return childRow, nil } return ProjectRow(ctx, i.projs, childRow) } From a617bc6416910aa885d4943e1a3c2ccf23d20774 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 30 Sep 2024 16:24:49 -0700 Subject: [PATCH 20/39] the same but not --- server/handler.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/server/handler.go b/server/handler.go index 8afa1eda84..909b623604 100644 --- a/server/handler.go +++ b/server/handler.go @@ -932,18 +932,28 @@ func RowToSQL(ctx *sql.Context, sch sql.Schema, row sql.Row, projs []sql.Express if len(sch) == 0 { return []sqltypes.Value{}, nil } - var field interface{} - var err error + outVals := make([]sqltypes.Value, len(sch)) - for i, col := range sch { - if len(projs) == 0 { - field = row[i] - } else { - field, err = projs[i].Eval(ctx, row) + if len(projs) == 0 { + for i, col := range sch { + if row[i] == nil { + outVals[i] = sqltypes.NULL + continue + } + var err error + outVals[i], err = col.Type.SQL(ctx, nil, row[i]) if err != nil { return nil, err } } + return outVals, nil + } + + for i, col := range sch { + field, err := projs[i].Eval(ctx, row) + if err != nil { + return nil, err + } if field == nil { outVals[i] = sqltypes.NULL continue From 693dfae4c0a30abe84f86070d2b4f3b9fecbc4a7 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 1 Oct 2024 12:03:45 -0700 Subject: [PATCH 21/39] unnecessary? --- server/handler.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/handler.go b/server/handler.go index 909b623604..b057f052c7 100644 --- a/server/handler.go +++ b/server/handler.go @@ -518,11 +518,6 @@ func GetDeferredProjections(iter sql.RowIter) []sql.Expression { case *rowexec.ExprCloserIter: return GetDeferredProjections(i.GetIter()) case *plan.TrackedRowIter: - // TODO: absolutely no idea why i.GetNode() is nil sometimes and why that means the deferred projections - // don't work but that is how it is - if i.GetNode() == nil { - return nil - } if commitIter, isCommitIter := i.GetIter().(*rowexec.TransactionCommittingIter); isCommitIter { if projIter, isProjIter := commitIter.GetIter().(*rowexec.ProjectIter); isProjIter { if projIter.CanDefer() { From 429ca08dd81355c45aef848641891bf443d883a6 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 1 Oct 2024 12:08:15 -0700 Subject: [PATCH 22/39] revert alloc --- sql/rowexec/rel_iters.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index a331278190..0d0e37e185 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -149,7 +149,7 @@ func ProjectRow( row sql.Row, ) (sql.Row, error) { var fields = make(sql.Row, len(projections)) - var secondPass = make([]int, 0, len(projections)) + var secondPass []int for i, expr := range projections { // Default values that are expressions may reference other fields, thus they must evaluate after all other exprs. // Also default expressions may not refer to other columns that come after them if they also have a default expr. From c19546c89fe902c8659404352aca30e4790e6c9b Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 1 Oct 2024 12:54:02 -0700 Subject: [PATCH 23/39] fix --- server/handler.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/handler.go b/server/handler.go index b057f052c7..35bbd08dc7 100644 --- a/server/handler.go +++ b/server/handler.go @@ -533,6 +533,7 @@ func GetDeferredProjections(iter sql.RowIter) []sql.Expression { // resultForMax1RowIter ensures that an empty iterator returns at most one row func resultForMax1RowIter(ctx *sql.Context, schema sql.Schema, iter sql.RowIter, resultFields []*querypb.Field) (*sqltypes.Result, error) { defer trace.StartRegion(ctx, "Handler.resultForMax1RowIter").End() + projs := GetDeferredProjections(iter) row, err := iter.Next(ctx) if err == io.EOF { return &sqltypes.Result{Fields: resultFields}, nil @@ -546,8 +547,6 @@ func resultForMax1RowIter(ctx *sql.Context, schema sql.Schema, iter sql.RowIter, if err := iter.Close(ctx); err != nil { return nil, err } - - projs := GetDeferredProjections(iter) outputRow, err := RowToSQL(ctx, schema, row, projs) if err != nil { return nil, err From 052c0d929995f40826e35a8d6622b1c8ba8beb46 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 1 Oct 2024 13:22:02 -0700 Subject: [PATCH 24/39] move poll --- server/handler.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/handler.go b/server/handler.go index 00cf8a162e..3d4ebef75b 100644 --- a/server/handler.go +++ b/server/handler.go @@ -558,12 +558,6 @@ func (h *Handler) resultForDefaultIter( } } - pollCtx, cancelF := ctx.NewSubContext() - eg.Go(func() error { - defer pan2err() - return h.pollForClosedConnection(pollCtx, c) - }) - wg := sync.WaitGroup{} wg.Add(2) @@ -594,6 +588,12 @@ func (h *Handler) resultForDefaultIter( } }) + pollCtx, cancelF := ctx.NewSubContext() + eg.Go(func() error { + defer pan2err() + return h.pollForClosedConnection(pollCtx, c) + }) + // Default waitTime is one minute if there is no timeout configured, in which case // it will loop to iterate again unless the socket died by the OS timeout or other problems. // If there is a timeout, it will be enforced to ensure that Vitess has a chance to From 78b297ff5932144c78409cc25c3d8494c790cc87 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 1 Oct 2024 13:27:05 -0700 Subject: [PATCH 25/39] readd extra row alloc --- sql/rowexec/rel_iters.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index 0d0e37e185..4397c1ea55 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -175,7 +175,7 @@ func ProjectRow( field = normalizeNegativeZeros(field) fields[index] = field } - return fields, nil + return sql.NewRow(fields...), nil } func defaultValFromProjectExpr(e sql.Expression) (*sql.ColumnDefaultValue, bool) { From 05f30a1a11f2705717f6bb25a198de4e1b8f3ee4 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 1 Oct 2024 13:37:17 -0700 Subject: [PATCH 26/39] undo --- sql/rowexec/rel_iters.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index 4397c1ea55..0d0e37e185 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -175,7 +175,7 @@ func ProjectRow( field = normalizeNegativeZeros(field) fields[index] = field } - return sql.NewRow(fields...), nil + return fields, nil } func defaultValFromProjectExpr(e sql.Expression) (*sql.ColumnDefaultValue, bool) { From 1ae4a8f59c17b91f54057dc27b332c017774b470 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 1 Oct 2024 16:24:35 -0700 Subject: [PATCH 27/39] skip opt for single results --- server/handler.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/handler.go b/server/handler.go index be2834fbef..69df9e2ce0 100644 --- a/server/handler.go +++ b/server/handler.go @@ -533,7 +533,6 @@ func GetDeferredProjections(iter sql.RowIter) []sql.Expression { // resultForMax1RowIter ensures that an empty iterator returns at most one row func resultForMax1RowIter(ctx *sql.Context, schema sql.Schema, iter sql.RowIter, resultFields []*querypb.Field) (*sqltypes.Result, error) { defer trace.StartRegion(ctx, "Handler.resultForMax1RowIter").End() - projs := GetDeferredProjections(iter) row, err := iter.Next(ctx) if err == io.EOF { return &sqltypes.Result{Fields: resultFields}, nil @@ -547,7 +546,7 @@ func resultForMax1RowIter(ctx *sql.Context, schema sql.Schema, iter sql.RowIter, if err := iter.Close(ctx); err != nil { return nil, err } - outputRow, err := RowToSQL(ctx, schema, row, projs) + outputRow, err := RowToSQL(ctx, schema, row, nil) if err != nil { return nil, err } From 69675990fa010b1f8c38ce977bfb9cd36d9b1445 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 2 Oct 2024 13:31:32 -0700 Subject: [PATCH 28/39] inline into planbuilder --- engine.go | 6 +++--- enginetest/evaluation.go | 2 +- enginetest/server_engine.go | 1 - server/handler.go | 11 +++++------ server/server.go | 1 - sql/analyzer/analyzer.go | 2 -- sql/analyzer/optimization_rules.go | 24 ------------------------ sql/analyzer/rules.go | 1 - sql/plan/project.go | 4 ++-- sql/planbuilder/from.go | 2 +- sql/planbuilder/parse.go | 6 ++++-- sql/planbuilder/parse_test.go | 2 +- sql/planbuilder/project.go | 22 ++++++++++++++++++++-- sql/query_flags.go | 2 ++ sql/rowexec/rel.go | 2 +- sql/rowexec/rel_iters.go | 9 ++------- 16 files changed, 42 insertions(+), 55 deletions(-) diff --git a/engine.go b/engine.go index cfdcbb32c5..126aad7877 100644 --- a/engine.go +++ b/engine.go @@ -238,7 +238,7 @@ func (e *Engine) PrepareParsedQuery( stmt sqlparser.Statement, ) (sql.Node, error) { binder := planbuilder.New(ctx, e.Analyzer.Catalog, e.Parser) - node, _, err := binder.BindOnly(stmt, query) + node, _, err := binder.BindOnly(stmt, query, nil) if err != nil { return nil, err @@ -595,7 +595,7 @@ func (e *Engine) bindQuery(ctx *sql.Context, query string, parsed sqlparser.Stat return nil, nil, err } } else { - bound, qFlags, err = binder.BindOnly(parsed, query) + bound, qFlags, err = binder.BindOnly(parsed, query, qFlags) if err != nil { return nil, nil, err } @@ -651,7 +651,7 @@ func (e *Engine) bindExecuteQueryNode(ctx *sql.Context, query string, eq *plan.E binder.SetBindingsWithExpr(tempBindings) } - bound, _, err := binder.BindOnly(prep, query) + bound, _, err := binder.BindOnly(prep, query, nil) if err != nil { clearAutocommitErr := clearAutocommitTransaction(ctx) if clearAutocommitErr != nil { diff --git a/enginetest/evaluation.go b/enginetest/evaluation.go index a99c9decd0..ef57d62dbb 100644 --- a/enginetest/evaluation.go +++ b/enginetest/evaluation.go @@ -528,7 +528,7 @@ func injectBindVarsAndPrepare( b := planbuilder.New(ctx, e.EngineAnalyzer().Catalog, sql.NewMysqlParser()) b.SetParserOptions(sql.LoadSqlMode(ctx).ParserOptions()) - resPlan, _, err := b.BindOnly(parsed, q) + resPlan, _, err := b.BindOnly(parsed, q, nil) if err != nil { return q, nil, err } diff --git a/enginetest/server_engine.go b/enginetest/server_engine.go index 8f59429ee9..da7f22e03a 100644 --- a/enginetest/server_engine.go +++ b/enginetest/server_engine.go @@ -78,7 +78,6 @@ func NewServerQueryEngine(t *testing.T, engine *sqle.Engine, builder server.Sess _ = s.Start() }() - engine.Analyzer.ServerMode = true return &ServerQueryEngine{ t: t, engine: engine, diff --git a/server/handler.go b/server/handler.go index 69df9e2ce0..5aae08f5fe 100644 --- a/server/handler.go +++ b/server/handler.go @@ -219,7 +219,7 @@ func (h *Handler) ComExecuteBound(ctx context.Context, conn *mysql.Conn, query s func (h *Handler) ComStmtExecute(ctx context.Context, c *mysql.Conn, prepare *mysql.PrepareData, callback func(*sqltypes.Result) error) error { _, err := h.errorWrappedDoQuery(ctx, c, prepare.PrepareStmt, nil, MultiStmtModeOff, prepare.BindVars, func(res *sqltypes.Result, more bool) error { return callback(res) - }, nil) + }, &sql.QueryFlags{}) return err } @@ -296,7 +296,7 @@ func (h *Handler) ComMultiQuery( query string, callback mysql.ResultSpoolFn, ) (string, error) { - return h.errorWrappedDoQuery(ctx, c, query, nil, MultiStmtModeOn, nil, callback, nil) + return h.errorWrappedDoQuery(ctx, c, query, nil, MultiStmtModeOn, nil, callback, &sql.QueryFlags{}) } // ComQuery executes a SQL query on the SQLe engine. @@ -306,7 +306,7 @@ func (h *Handler) ComQuery( query string, callback mysql.ResultSpoolFn, ) error { - _, err := h.errorWrappedDoQuery(ctx, c, query, nil, MultiStmtModeOff, nil, callback, nil) + _, err := h.errorWrappedDoQuery(ctx, c, query, nil, MultiStmtModeOff, nil, callback, &sql.QueryFlags{}) return err } @@ -318,7 +318,7 @@ func (h *Handler) ComParsedQuery( parsed sqlparser.Statement, callback mysql.ResultSpoolFn, ) error { - _, err := h.errorWrappedDoQuery(ctx, c, query, parsed, MultiStmtModeOff, nil, callback, nil) + _, err := h.errorWrappedDoQuery(ctx, c, query, parsed, MultiStmtModeOff, nil, callback, &sql.QueryFlags{}) return err } @@ -520,8 +520,7 @@ func GetDeferredProjections(iter sql.RowIter) []sql.Expression { case *plan.TrackedRowIter: if commitIter, isCommitIter := i.GetIter().(*rowexec.TransactionCommittingIter); isCommitIter { if projIter, isProjIter := commitIter.GetIter().(*rowexec.ProjectIter); isProjIter { - if projIter.CanDefer() { - projIter.Defer() + if projIter.Deferred() { return projIter.GetProjections() } } diff --git a/server/server.go b/server/server.go index 12e7820aaf..e77a933253 100644 --- a/server/server.go +++ b/server/server.go @@ -182,7 +182,6 @@ func newServerFromHandler(cfg Config, e *sqle.Engine, sm *SessionManager, handle vtListener.RequireSecureTransport = cfg.RequireSecureTransport } - e.Analyzer.ServerMode = true return &Server{ Listener: protocolListener, handler: handler, diff --git a/sql/analyzer/analyzer.go b/sql/analyzer/analyzer.go index da20fa3130..8a2834ecbb 100644 --- a/sql/analyzer/analyzer.go +++ b/sql/analyzer/analyzer.go @@ -298,8 +298,6 @@ type Analyzer struct { // EventScheduler is used to communiate with the event scheduler // for any EVENT related statements. It can be nil if EventScheduler is not defined. EventScheduler sql.EventScheduler - // ServerMode is true if the analyzer is running in server mode. - ServerMode bool } // NewDefault creates a default Analyzer instance with all default Rules and configuration. diff --git a/sql/analyzer/optimization_rules.go b/sql/analyzer/optimization_rules.go index 15f9ba63eb..cb20012c91 100644 --- a/sql/analyzer/optimization_rules.go +++ b/sql/analyzer/optimization_rules.go @@ -19,7 +19,6 @@ import ( "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/expression" - "github.com/dolthub/go-mysql-server/sql/expression/function" "github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/transform" "github.com/dolthub/go-mysql-server/sql/types" @@ -55,29 +54,6 @@ func eraseProjection(ctx *sql.Context, a *Analyzer, node sql.Node, scope *plan.S }) } -// deferProjections marks projection nodes that can defer projections to the end of the query execution. -// This can avoid an unnecessary slice allocation specifically in the case where we are spooling rows -// from the server to the client. -func deferProjections(ctx *sql.Context, a *Analyzer, node sql.Node, scope *plan.Scope, sel RuleSelector, qFlags *sql.QueryFlags) (sql.Node, transform.TreeIdentity, error) { - if !a.ServerMode { - return node, transform.SameTree, nil - } - // Only defer top-level projections - proj, isProj := node.(*plan.Project) - if !isProj { - return node, transform.SameTree, nil - } - // Default value expressions require a second pass, so punt on deferring for now - for _, expr := range proj.Projections { - switch expr.(type) { - case *expression.Wrapper, *sql.ColumnDefaultValue, function.RowCount: - return node, transform.SameTree, nil - } - } - newProj := proj.WithCanDefer(true) - return newProj, transform.NewTree, nil -} - func flattenDistinct(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.Scope, sel RuleSelector, qFlags *sql.QueryFlags) (sql.Node, transform.TreeIdentity, error) { return transform.Node(n, func(n sql.Node) (sql.Node, transform.TreeIdentity, error) { if d, ok := n.(*plan.Distinct); ok { diff --git a/sql/analyzer/rules.go b/sql/analyzer/rules.go index dc9d1ce711..dbfd801e5a 100644 --- a/sql/analyzer/rules.go +++ b/sql/analyzer/rules.go @@ -26,7 +26,6 @@ func init() { {inlineSubqueryAliasRefsId, inlineSubqueryAliasRefs}, {cacheSubqueryAliasesInJoinsId, cacheSubqueryAliasesInJoins}, {backtickDefaulColumnValueNamesId, backtickDefaultColumnValueNames}, - {DeferProjectionsId, deferProjections}, {AutocommitId, addAutocommitNode}, {TrackProcessId, trackProcess}, {parallelizeId, parallelize}, diff --git a/sql/plan/project.go b/sql/plan/project.go index 814f63bd13..a3cabe4bc9 100644 --- a/sql/plan/project.go +++ b/sql/plan/project.go @@ -27,7 +27,7 @@ import ( type Project struct { UnaryNode Projections []sql.Expression - CanDefer bool + Deferred bool } var _ sql.Expressioner = (*Project)(nil) @@ -187,6 +187,6 @@ func (p *Project) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { func (p *Project) WithCanDefer(canDefer bool) *Project { np := *p - np.CanDefer = canDefer + np.Deferred = canDefer return &np } diff --git a/sql/planbuilder/from.go b/sql/planbuilder/from.go index c466c0bac7..5bbd5e156c 100644 --- a/sql/planbuilder/from.go +++ b/sql/planbuilder/from.go @@ -888,5 +888,5 @@ func (b *Builder) bindOnlyWithDatabase(db sql.Database, stmt ast.Statement, s st b.currentDatabase = curDb }() b.currentDatabase = db - return b.BindOnly(stmt, s) + return b.BindOnly(stmt, s, nil) } diff --git a/sql/planbuilder/parse.go b/sql/planbuilder/parse.go index e5df71a096..94b7ab510c 100644 --- a/sql/planbuilder/parse.go +++ b/sql/planbuilder/parse.go @@ -79,7 +79,7 @@ func (b *Builder) Parse(query string, multi bool) (ret sql.Node, parsed, remaind return outScope.node, parsed, remainder, b.qFlags, err } -func (b *Builder) BindOnly(stmt ast.Statement, s string) (_ sql.Node, _ *sql.QueryFlags, err error) { +func (b *Builder) BindOnly(stmt ast.Statement, s string, queryFlags *sql.QueryFlags) (_ sql.Node, _ *sql.QueryFlags, err error) { defer trace.StartRegion(b.ctx, "BindOnly").End() defer func() { if r := recover(); r != nil { @@ -91,7 +91,9 @@ func (b *Builder) BindOnly(stmt ast.Statement, s string) (_ sql.Node, _ *sql.Que } } }() - + if queryFlags != nil { + b.qFlags = queryFlags + } outScope := b.build(nil, stmt, s) return outScope.node, b.qFlags, err } diff --git a/sql/planbuilder/parse_test.go b/sql/planbuilder/parse_test.go index 0ab4554644..c0275977ec 100644 --- a/sql/planbuilder/parse_test.go +++ b/sql/planbuilder/parse_test.go @@ -2871,7 +2871,7 @@ func TestPlanBuilderErr(t *testing.T) { stmt, err := sqlparser.Parse(tt.Query) require.NoError(t, err) - _, _, err = b.BindOnly(stmt, tt.Query) + _, _, err = b.BindOnly(stmt, tt.Query, nil) defer b.Reset() require.Error(t, err) diff --git a/sql/planbuilder/project.go b/sql/planbuilder/project.go index ebac3c8b8d..0c277bf3fd 100644 --- a/sql/planbuilder/project.go +++ b/sql/planbuilder/project.go @@ -15,13 +15,14 @@ package planbuilder import ( - "strings" + "github.com/dolthub/go-mysql-server/sql/expression/function" +"strings" ast "github.com/dolthub/vitess/go/vt/sqlparser" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/expression" - "github.com/dolthub/go-mysql-server/sql/plan" + "github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/transform" ) @@ -204,6 +205,23 @@ func (b *Builder) buildProjection(inScope, outScope *scope) { b.handleErr(err) } outScope.node = proj + if !b.qFlags.IsSet(sql.QFlagDeferProjections) { + return + } + if inScope.parent != nil && inScope.parent.activeSubquery != nil { + return + } + if _, isProj := proj.(*plan.Project); !isProj { + return + } + for _, sc := range outScope.cols { + switch sc.scalar.(type) { + // TODO: column default expression are also not deferrable, but they don't appear in top level projections + case function.RowCount: + return + } + } + proj.(*plan.Project).Deferred = true } func selectExprNeedsAlias(e *ast.AliasedExpr, expr sql.Expression) bool { diff --git a/sql/query_flags.go b/sql/query_flags.go index 5d2ba22f05..526a97f724 100644 --- a/sql/query_flags.go +++ b/sql/query_flags.go @@ -42,6 +42,8 @@ const ( // QFlagMax1Row indicates that a query can only return at most one row QFlagMax1Row + + QFlagDeferProjections ) type QueryFlags struct { diff --git a/sql/rowexec/rel.go b/sql/rowexec/rel.go index 17bb1c7f37..eb74a94e41 100644 --- a/sql/rowexec/rel.go +++ b/sql/rowexec/rel.go @@ -313,7 +313,7 @@ func (b *BaseBuilder) buildProject(ctx *sql.Context, n *plan.Project, row sql.Ro return sql.NewSpanIter(span, &ProjectIter{ projs: n.Projections, - canDefer: n.CanDefer, + deferred: n.Deferred, childIter: i, }), nil } diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index 7569306e19..13e12a4ed1 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -126,7 +126,6 @@ var _ sql.RowIter = &iters.JsonTableRowIter{} type ProjectIter struct { projs []sql.Expression - canDefer bool deferred bool childIter sql.RowIter } @@ -150,12 +149,8 @@ func (i *ProjectIter) GetProjections() []sql.Expression { return i.projs } -func (i *ProjectIter) CanDefer() bool { - return i.canDefer -} - -func (i *ProjectIter) Defer() { - i.deferred = true +func (i *ProjectIter) Deferred() bool { + return i.deferred } // ProjectRow evaluates a set of projections. From 70de43b16a4d8becd125fd07100851c12cec974a Mon Sep 17 00:00:00 2001 From: jycor Date: Wed, 2 Oct 2024 20:34:03 +0000 Subject: [PATCH 29/39] [ga-format-pr] Run ./format_repo.sh to fix formatting --- sql/planbuilder/project.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/planbuilder/project.go b/sql/planbuilder/project.go index 0c277bf3fd..d70bc8e042 100644 --- a/sql/planbuilder/project.go +++ b/sql/planbuilder/project.go @@ -15,14 +15,14 @@ package planbuilder import ( - "github.com/dolthub/go-mysql-server/sql/expression/function" -"strings" + "strings" ast "github.com/dolthub/vitess/go/vt/sqlparser" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/expression" - "github.com/dolthub/go-mysql-server/sql/plan" + "github.com/dolthub/go-mysql-server/sql/expression/function" + "github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/transform" ) From 197ff0d0d392ac32f0639983123b111ee17818b3 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 2 Oct 2024 14:03:06 -0700 Subject: [PATCH 30/39] tidying --- sql/analyzer/rule_ids.go | 1 - sql/analyzer/ruleid_string.go | 13 ++++++------- sql/query_flags.go | 2 ++ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sql/analyzer/rule_ids.go b/sql/analyzer/rule_ids.go index 4936e23958..a99404d087 100644 --- a/sql/analyzer/rule_ids.go +++ b/sql/analyzer/rule_ids.go @@ -142,7 +142,6 @@ const ( cacheSubqueryResultsId // cacheSubqueryResults cacheSubqueryAliasesInJoinsId // cacheSubqueryAliasesInJoins backtickDefaulColumnValueNamesId // backtickDefaulColumnValueNames - DeferProjectionsId // deferProjections AutocommitId // addAutocommitNode TrackProcessId // trackProcess parallelizeId // parallelize diff --git a/sql/analyzer/ruleid_string.go b/sql/analyzer/ruleid_string.go index 7a9f5c73b3..af9abbb3ae 100755 --- a/sql/analyzer/ruleid_string.go +++ b/sql/analyzer/ruleid_string.go @@ -136,16 +136,15 @@ func _() { _ = x[cacheSubqueryResultsId-125] _ = x[cacheSubqueryAliasesInJoinsId-126] _ = x[backtickDefaulColumnValueNamesId-127] - _ = x[DeferProjectionsId-128] - _ = x[AutocommitId-129] - _ = x[TrackProcessId-130] - _ = x[parallelizeId-131] - _ = x[clearWarningsId-132] + _ = x[AutocommitId-128] + _ = x[TrackProcessId-129] + _ = x[parallelizeId-130] + _ = x[clearWarningsId-131] } -const _RuleId_name = "applyDefaultSelectLimitvalidateOffsetAndLimitvalidateStarExpressionsvalidateCreateTablevalidateAlterTablevalidateExprSemresolveVariablesresolveNamedWindowsresolveSetVariablesresolveViewsliftCtesresolveCtesliftRecursiveCtesresolveDatabasesresolveTablesloadStoredProceduresvalidateDropTablespruneDropTablessetTargetSchemasresolveCreateLikeparseColumnDefaultsresolveDropConstraintvalidateDropConstraintloadCheckConstraintsassignCatalogresolveAnalyzeTablesresolveCreateSelectresolveSubqueriessetViewTargetSchemaresolveUnionsresolveDescribeQuerycheckUniqueTableNamesresolveTableFunctionsresolveDeclarationsresolveColumnDefaultsvalidateColumnDefaultsvalidateCreateTriggervalidateCreateProcedureresolveCreateProcedureloadInfoSchemavalidateReadOnlyDatabasevalidateReadOnlyTransactionvalidateDatabaseSetvalidatePrivilegesreresolveTablessetInsertColumnsvalidateJoinComplexityapplyBinlogReplicaControllerapplyEventSchedulerresolveUsingJoinsresolveOrderbyLiteralsresolveFunctionsflattenTableAliasespushdownSortpushdownGroupbyAliasespushdownSubqueryAliasFiltersqualifyColumnsresolveColumnsvalidateCheckConstraintresolveBarewordSetVariablesreplaceCountStarexpandStarstransposeRightJoinsresolveHavingmergeUnionSchemasflattenAggregationExprsreorderProjectionresolveSubqueryExprsreplaceCrossJoinsmoveJoinCondsToFiltermoveFiltersToJoinCondsimplifyFilterspushNotFiltersoptimizeDistincthoistOutOfScopeFiltersunnestInSubqueriesunnestExistsSubqueriesfinalizeSubqueriesfinalizeUnionsloadTriggersloadEventsprocessTruncateresolveAlterColumnresolveGeneratorsremoveUnnecessaryConvertsstripTableNamesFromColumnDefaultsfoldEmptyJoinsoptimizeJoinsgenerateIndexScansmatchAgainstpushFiltersapplyIndexesFromOuterScopepruneTablesfixupAuxiliaryExprsassignExecIndexesinlineSubqueryAliasRefseraseProjectionflattenDistinctreplaceAggreplaceIdxSortinsertTopNreplaceIdxOrderByDistanceapplyHashInresolveInsertRowsresolvePreparedInsertapplyTriggersapplyProceduresassignRoutinesmodifyUpdateExprsForJoinapplyRowUpdateAccumulatorsrollback triggersapplyFKsvalidateResolvedvalidateOrderByvalidateGroupByvalidateSchemaSourcevalidateIndexCreationvalidateOperandsvalidateCaseResultTypesvalidateIntervalUsagevalidateExplodeUsagevalidateSubqueryColumnsvalidateUnionSchemasMatchvalidateAggregationsvalidateDeleteFromcacheSubqueryResultscacheSubqueryAliasesInJoinsbacktickDefaulColumnValueNamesdeferProjectionsaddAutocommitNodetrackProcessparallelizeclearWarnings" +const _RuleId_name = "applyDefaultSelectLimitvalidateOffsetAndLimitvalidateStarExpressionsvalidateCreateTablevalidateAlterTablevalidateExprSemresolveVariablesresolveNamedWindowsresolveSetVariablesresolveViewsliftCtesresolveCtesliftRecursiveCtesresolveDatabasesresolveTablesloadStoredProceduresvalidateDropTablespruneDropTablessetTargetSchemasresolveCreateLikeparseColumnDefaultsresolveDropConstraintvalidateDropConstraintloadCheckConstraintsassignCatalogresolveAnalyzeTablesresolveCreateSelectresolveSubqueriessetViewTargetSchemaresolveUnionsresolveDescribeQuerycheckUniqueTableNamesresolveTableFunctionsresolveDeclarationsresolveColumnDefaultsvalidateColumnDefaultsvalidateCreateTriggervalidateCreateProcedureresolveCreateProcedureloadInfoSchemavalidateReadOnlyDatabasevalidateReadOnlyTransactionvalidateDatabaseSetvalidatePrivilegesreresolveTablessetInsertColumnsvalidateJoinComplexityapplyBinlogReplicaControllerapplyEventSchedulerresolveUsingJoinsresolveOrderbyLiteralsresolveFunctionsflattenTableAliasespushdownSortpushdownGroupbyAliasespushdownSubqueryAliasFiltersqualifyColumnsresolveColumnsvalidateCheckConstraintresolveBarewordSetVariablesreplaceCountStarexpandStarstransposeRightJoinsresolveHavingmergeUnionSchemasflattenAggregationExprsreorderProjectionresolveSubqueryExprsreplaceCrossJoinsmoveJoinCondsToFiltermoveFiltersToJoinCondsimplifyFilterspushNotFiltersoptimizeDistincthoistOutOfScopeFiltersunnestInSubqueriesunnestExistsSubqueriesfinalizeSubqueriesfinalizeUnionsloadTriggersloadEventsprocessTruncateresolveAlterColumnresolveGeneratorsremoveUnnecessaryConvertsstripTableNamesFromColumnDefaultsfoldEmptyJoinsoptimizeJoinsgenerateIndexScansmatchAgainstpushFiltersapplyIndexesFromOuterScopepruneTablesfixupAuxiliaryExprsassignExecIndexesinlineSubqueryAliasRefseraseProjectionflattenDistinctreplaceAggreplaceIdxSortinsertTopNreplaceIdxOrderByDistanceapplyHashInresolveInsertRowsresolvePreparedInsertapplyTriggersapplyProceduresassignRoutinesmodifyUpdateExprsForJoinapplyRowUpdateAccumulatorsrollback triggersapplyFKsvalidateResolvedvalidateOrderByvalidateGroupByvalidateSchemaSourcevalidateIndexCreationvalidateOperandsvalidateCaseResultTypesvalidateIntervalUsagevalidateExplodeUsagevalidateSubqueryColumnsvalidateUnionSchemasMatchvalidateAggregationsvalidateDeleteFromcacheSubqueryResultscacheSubqueryAliasesInJoinsbacktickDefaulColumnValueNamesaddAutocommitNodetrackProcessparallelizeclearWarnings" -var _RuleId_index = [...]uint16{0, 23, 45, 68, 87, 105, 120, 136, 155, 174, 186, 194, 205, 222, 238, 251, 271, 289, 304, 320, 337, 356, 377, 399, 419, 432, 452, 471, 488, 507, 520, 540, 561, 582, 601, 622, 644, 665, 688, 710, 724, 748, 775, 794, 812, 827, 843, 865, 893, 912, 929, 951, 967, 986, 998, 1020, 1048, 1062, 1076, 1099, 1126, 1142, 1153, 1172, 1185, 1202, 1225, 1242, 1262, 1279, 1300, 1321, 1336, 1350, 1366, 1388, 1406, 1428, 1446, 1460, 1472, 1482, 1497, 1515, 1532, 1557, 1590, 1604, 1617, 1635, 1647, 1658, 1684, 1695, 1714, 1731, 1754, 1769, 1784, 1794, 1808, 1818, 1843, 1854, 1871, 1892, 1905, 1920, 1934, 1958, 1984, 2001, 2009, 2025, 2040, 2055, 2075, 2096, 2112, 2135, 2156, 2176, 2199, 2224, 2244, 2262, 2282, 2309, 2339, 2355, 2372, 2384, 2395, 2408} +var _RuleId_index = [...]uint16{0, 23, 45, 68, 87, 105, 120, 136, 155, 174, 186, 194, 205, 222, 238, 251, 271, 289, 304, 320, 337, 356, 377, 399, 419, 432, 452, 471, 488, 507, 520, 540, 561, 582, 601, 622, 644, 665, 688, 710, 724, 748, 775, 794, 812, 827, 843, 865, 893, 912, 929, 951, 967, 986, 998, 1020, 1048, 1062, 1076, 1099, 1126, 1142, 1153, 1172, 1185, 1202, 1225, 1242, 1262, 1279, 1300, 1321, 1336, 1350, 1366, 1388, 1406, 1428, 1446, 1460, 1472, 1482, 1497, 1515, 1532, 1557, 1590, 1604, 1617, 1635, 1647, 1658, 1684, 1695, 1714, 1731, 1754, 1769, 1784, 1794, 1808, 1818, 1843, 1854, 1871, 1892, 1905, 1920, 1934, 1958, 1984, 2001, 2009, 2025, 2040, 2055, 2075, 2096, 2112, 2135, 2156, 2176, 2199, 2224, 2244, 2262, 2282, 2309, 2339, 2356, 2368, 2379, 2392} func (i RuleId) String() string { if i < 0 || i >= RuleId(len(_RuleId_index)-1) { diff --git a/sql/query_flags.go b/sql/query_flags.go index 526a97f724..86c9310dff 100644 --- a/sql/query_flags.go +++ b/sql/query_flags.go @@ -43,6 +43,8 @@ const ( // QFlagMax1Row indicates that a query can only return at most one row QFlagMax1Row + // QFlagDeferProjections indicates that a top-level projections for this query should be deferred and handled by + // RowToSQL QFlagDeferProjections ) From a16c2d2f4d02c854fce967a2fe7dae29306dc9da Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 3 Oct 2024 01:44:36 -0700 Subject: [PATCH 31/39] real opt --- engine.go | 4 ++-- enginetest/engine_only_test.go | 2 +- server/handler.go | 4 +++- sql/analyzer/stored_procedures.go | 4 ++-- sql/analyzer/triggers.go | 4 ++-- sql/plan/project.go | 4 ++-- sql/planbuilder/parse.go | 8 ++++++-- sql/planbuilder/project.go | 2 +- sql/planbuilder/show.go | 10 +++++----- sql/rowexec/rel.go | 2 +- sql/rowexec/rel_iters.go | 9 +++++++-- 11 files changed, 32 insertions(+), 21 deletions(-) diff --git a/engine.go b/engine.go index 126aad7877..a8a8988f26 100644 --- a/engine.go +++ b/engine.go @@ -210,7 +210,7 @@ func (e *Engine) AnalyzeQuery( query string, ) (sql.Node, error) { binder := planbuilder.New(ctx, e.Analyzer.Catalog, e.Parser) - parsed, _, _, qFlags, err := binder.Parse(query, false) + parsed, _, _, qFlags, err := binder.Parse(query, nil, false) if err != nil { return nil, err } @@ -586,7 +586,7 @@ func (e *Engine) bindQuery(ctx *sql.Context, query string, parsed sqlparser.Stat var bound sql.Node var err error if parsed == nil { - bound, _, _, qFlags, err = binder.Parse(query, false) + bound, _, _, qFlags, err = binder.Parse(query, qFlags, false) if err != nil { clearAutocommitErr := clearAutocommitTransaction(ctx) if clearAutocommitErr != nil { diff --git a/enginetest/engine_only_test.go b/enginetest/engine_only_test.go index 6fb9c344a7..92c16714f0 100644 --- a/enginetest/engine_only_test.go +++ b/enginetest/engine_only_test.go @@ -511,7 +511,7 @@ func TestAnalyzer_Exp(t *testing.T) { ctx := enginetest.NewContext(harness) b := planbuilder.New(ctx, e.EngineAnalyzer().Catalog, sql.NewMysqlParser()) - parsed, _, _, _, err := b.Parse(tt.query, false) + parsed, _, _, _, err := b.Parse(tt.query, nil, false) require.NoError(t, err) analyzed, err := e.EngineAnalyzer().Analyze(ctx, parsed, nil, nil) diff --git a/server/handler.go b/server/handler.go index 5aae08f5fe..f72336d276 100644 --- a/server/handler.go +++ b/server/handler.go @@ -425,6 +425,7 @@ func (h *Handler) doQuery( } }() + qFlags.Set(sql.QFlagDeferProjections) schema, rowIter, qFlags, err := queryExec(sqlCtx, query, parsed, analyzedPlan, bindings, qFlags) if err != nil { sqlCtx.GetLogger().WithError(err).Warn("error running query") @@ -520,7 +521,8 @@ func GetDeferredProjections(iter sql.RowIter) []sql.Expression { case *plan.TrackedRowIter: if commitIter, isCommitIter := i.GetIter().(*rowexec.TransactionCommittingIter); isCommitIter { if projIter, isProjIter := commitIter.GetIter().(*rowexec.ProjectIter); isProjIter { - if projIter.Deferred() { + if projIter.CanDefer() { + projIter.Defer() return projIter.GetProjections() } } diff --git a/sql/analyzer/stored_procedures.go b/sql/analyzer/stored_procedures.go index f8a2750983..d1b1575b80 100644 --- a/sql/analyzer/stored_procedures.go +++ b/sql/analyzer/stored_procedures.go @@ -56,7 +56,7 @@ func loadStoredProcedures(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan var parsedProcedure sql.Node b := planbuilder.New(ctx, a.Catalog, sql.NewMysqlParser()) b.SetParserOptions(sql.NewSqlModeFromString(procedure.SqlMode).ParserOptions()) - parsedProcedure, _, _, _, err = b.Parse(procedure.CreateStatement, false) + parsedProcedure, _, _, _, err = b.Parse(procedure.CreateStatement, nil, false) if err != nil { procToRegister = &plan.Procedure{ CreateProcedureString: procedure.CreateStatement, @@ -300,7 +300,7 @@ func applyProcedures(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.Scop b.ProcCtx().AsOf = asOf } b.ProcCtx().DbName = call.Database().Name() - parsedProcedure, _, _, _, err = b.Parse(procedure.CreateStatement, false) + parsedProcedure, _, _, _, err = b.Parse(procedure.CreateStatement, nil, false) if err != nil { return nil, transform.SameTree, err } diff --git a/sql/analyzer/triggers.go b/sql/analyzer/triggers.go index 9bb6a5da45..4fe8d89db1 100644 --- a/sql/analyzer/triggers.go +++ b/sql/analyzer/triggers.go @@ -204,7 +204,7 @@ func applyTriggers(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.Scope, var parsedTrigger sql.Node sqlMode := sql.NewSqlModeFromString(trigger.SqlMode) b.SetParserOptions(sqlMode.ParserOptions()) - parsedTrigger, _, _, _, err = b.Parse(trigger.CreateStatement, false) + parsedTrigger, _, _, _, err = b.Parse(trigger.CreateStatement, nil, false) b.Reset() if err != nil { return nil, transform.SameTree, err @@ -225,7 +225,7 @@ func applyTriggers(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.Scope, // first pass allows unresolved before we know whether trigger is relevant // TODO store destination table name with trigger, so we don't have to do parse twice b.TriggerCtx().Call = true - parsedTrigger, _, _, _, err = b.Parse(trigger.CreateStatement, false) + parsedTrigger, _, _, _, err = b.Parse(trigger.CreateStatement, nil, false) b.TriggerCtx().Call = false b.Reset() if err != nil { diff --git a/sql/plan/project.go b/sql/plan/project.go index a3cabe4bc9..814f63bd13 100644 --- a/sql/plan/project.go +++ b/sql/plan/project.go @@ -27,7 +27,7 @@ import ( type Project struct { UnaryNode Projections []sql.Expression - Deferred bool + CanDefer bool } var _ sql.Expressioner = (*Project)(nil) @@ -187,6 +187,6 @@ func (p *Project) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { func (p *Project) WithCanDefer(canDefer bool) *Project { np := *p - np.Deferred = canDefer + np.CanDefer = canDefer return &np } diff --git a/sql/planbuilder/parse.go b/sql/planbuilder/parse.go index 94b7ab510c..58550301c5 100644 --- a/sql/planbuilder/parse.go +++ b/sql/planbuilder/parse.go @@ -41,11 +41,11 @@ func ParseWithOptions(ctx *sql.Context, cat sql.Catalog, query string, options a // TODO: need correct parser b := New(ctx, cat, sql.NewMysqlParser()) b.SetParserOptions(options) - node, _, _, qFlags, err := b.Parse(query, false) + node, _, _, qFlags, err := b.Parse(query, nil, false) return node, qFlags, err } -func (b *Builder) Parse(query string, multi bool) (ret sql.Node, parsed, remainder string, qProps *sql.QueryFlags, err error) { +func (b *Builder) Parse(query string, qFlags *sql.QueryFlags, multi bool) (ret sql.Node, parsed, remainder string, qProps *sql.QueryFlags, err error) { defer trace.StartRegion(b.ctx, "ParseOnly").End() b.nesting++ if b.nesting > maxAnalysisIterations { @@ -74,6 +74,10 @@ func (b *Builder) Parse(query string, multi bool) (ret sql.Node, parsed, remaind return nil, parsed, remainder, nil, sql.ErrSyntaxError.New(err.Error()) } + if qFlags != nil { + b.qFlags = qFlags + } + outScope := b.build(nil, stmt, parsed) return outScope.node, parsed, remainder, b.qFlags, err diff --git a/sql/planbuilder/project.go b/sql/planbuilder/project.go index d70bc8e042..95d98f5a33 100644 --- a/sql/planbuilder/project.go +++ b/sql/planbuilder/project.go @@ -221,7 +221,7 @@ func (b *Builder) buildProjection(inScope, outScope *scope) { return } } - proj.(*plan.Project).Deferred = true + proj.(*plan.Project).CanDefer = true } func selectExprNeedsAlias(e *ast.AliasedExpr, expr sql.Expression) bool { diff --git a/sql/planbuilder/show.go b/sql/planbuilder/show.go index d56fb30e1b..d5ee9c076e 100644 --- a/sql/planbuilder/show.go +++ b/sql/planbuilder/show.go @@ -343,7 +343,7 @@ func (b *Builder) buildShowProcedureStatus(inScope *scope, s *ast.Show) (outScop node, _, _, _, err := b.Parse("select routine_schema as `Db`, routine_name as `Name`, routine_type as `Type`,"+ "definer as `Definer`, last_altered as `Modified`, created as `Created`, security_type as `Security_type`,"+ "routine_comment as `Comment`, CHARACTER_SET_CLIENT as `character_set_client`, COLLATION_CONNECTION as `collation_connection`,"+ - "database_collation as `Database Collation` from information_schema.routines where routine_type = 'PROCEDURE'", false) + "database_collation as `Database Collation` from information_schema.routines where routine_type = 'PROCEDURE'", nil, false) if err != nil { b.handleErr(err) } @@ -379,7 +379,7 @@ func (b *Builder) buildShowFunctionStatus(inScope *scope, s *ast.Show) (outScope node, _, _, _, err := b.Parse("select routine_schema as `Db`, routine_name as `Name`, routine_type as `Type`,"+ "definer as `Definer`, last_altered as `Modified`, created as `Created`, security_type as `Security_type`,"+ "routine_comment as `Comment`, character_set_client, collation_connection,"+ - "database_collation as `Database Collation` from information_schema.routines where routine_type = 'FUNCTION'", false) + "database_collation as `Database Collation` from information_schema.routines where routine_type = 'FUNCTION'", nil, false) if err != nil { b.handleErr(err) } @@ -783,7 +783,7 @@ func (b *Builder) buildShowCollation(inScope *scope, s *ast.Show) (outScope *sco // information_schema, with slightly different syntax and with some columns aliased. // TODO: install information_schema automatically for all catalogs node, _, _, _, err := b.Parse("select collation_name as `collation`, character_set_name as charset, id,"+ - "is_default as `default`, is_compiled as compiled, sortlen, pad_attribute from information_schema.collations order by collation_name", false) + "is_default as `default`, is_compiled as compiled, sortlen, pad_attribute from information_schema.collations order by collation_name", nil, false) if err != nil { b.handleErr(err) } @@ -828,7 +828,7 @@ select XA as XA, SAVEPOINTS as Savepoints from information_schema.engines -`, false) +`, nil, false) if err != nil { b.handleErr(err) } @@ -839,7 +839,7 @@ from information_schema.engines func (b *Builder) buildShowPlugins(inScope *scope, s *ast.Show) (outScope *scope) { outScope = inScope.push() - infoSchemaSelect, _, _, _, err := b.Parse("select * from information_schema.plugins", false) + infoSchemaSelect, _, _, _, err := b.Parse("select * from information_schema.plugins", nil,false) if err != nil { b.handleErr(err) } diff --git a/sql/rowexec/rel.go b/sql/rowexec/rel.go index eb74a94e41..17bb1c7f37 100644 --- a/sql/rowexec/rel.go +++ b/sql/rowexec/rel.go @@ -313,7 +313,7 @@ func (b *BaseBuilder) buildProject(ctx *sql.Context, n *plan.Project, row sql.Ro return sql.NewSpanIter(span, &ProjectIter{ projs: n.Projections, - deferred: n.Deferred, + canDefer: n.CanDefer, childIter: i, }), nil } diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index 13e12a4ed1..7569306e19 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -126,6 +126,7 @@ var _ sql.RowIter = &iters.JsonTableRowIter{} type ProjectIter struct { projs []sql.Expression + canDefer bool deferred bool childIter sql.RowIter } @@ -149,8 +150,12 @@ func (i *ProjectIter) GetProjections() []sql.Expression { return i.projs } -func (i *ProjectIter) Deferred() bool { - return i.deferred +func (i *ProjectIter) CanDefer() bool { + return i.canDefer +} + +func (i *ProjectIter) Defer() { + i.deferred = true } // ProjectRow evaluates a set of projections. From 4b30e062e0af4ed74b218274c776b49694c4558b Mon Sep 17 00:00:00 2001 From: jycor Date: Thu, 3 Oct 2024 08:46:00 +0000 Subject: [PATCH 32/39] [ga-format-pr] Run ./format_repo.sh to fix formatting --- sql/planbuilder/show.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/planbuilder/show.go b/sql/planbuilder/show.go index d5ee9c076e..418235a2a7 100644 --- a/sql/planbuilder/show.go +++ b/sql/planbuilder/show.go @@ -839,7 +839,7 @@ from information_schema.engines func (b *Builder) buildShowPlugins(inScope *scope, s *ast.Show) (outScope *scope) { outScope = inScope.push() - infoSchemaSelect, _, _, _, err := b.Parse("select * from information_schema.plugins", nil,false) + infoSchemaSelect, _, _, _, err := b.Parse("select * from information_schema.plugins", nil, false) if err != nil { b.handleErr(err) } From 55df1f59712628f497caa36d426d5e08da07d956 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 3 Oct 2024 01:51:33 -0700 Subject: [PATCH 33/39] fix tests --- enginetest/plangen/cmd/plangen/main.go | 2 +- sql/analyzer/optimization_rules_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/enginetest/plangen/cmd/plangen/main.go b/enginetest/plangen/cmd/plangen/main.go index 30d248a2af..7779baf30b 100644 --- a/enginetest/plangen/cmd/plangen/main.go +++ b/enginetest/plangen/cmd/plangen/main.go @@ -166,7 +166,7 @@ func generatePlansForSuite(spec PlanSpec, w *bytes.Buffer) error { if !tt.Skip { ctx := enginetest.NewContextWithEngine(harness, engine) binder := planbuilder.New(ctx, engine.EngineAnalyzer().Catalog, sql.NewMysqlParser()) - parsed, _, _, qFlags, err := binder.Parse(tt.Query, false) + parsed, _, _, qFlags, err := binder.Parse(tt.Query, nil, false) if err != nil { exit(fmt.Errorf("%w\nfailed to parse query: %s", err, tt.Query)) } diff --git a/sql/analyzer/optimization_rules_test.go b/sql/analyzer/optimization_rules_test.go index 8b6a45d1ff..f36ee26fb7 100644 --- a/sql/analyzer/optimization_rules_test.go +++ b/sql/analyzer/optimization_rules_test.go @@ -223,7 +223,7 @@ func TestPushNotFilters(t *testing.T) { for _, tt := range tests { t.Run(tt.in, func(t *testing.T) { q := fmt.Sprintf("SELECT 1 from xy WHERE %s", tt.in) - node, _, _, _, err := b.Parse(q, false) + node, _, _, _, err := b.Parse(q, nil, false) require.NoError(t, err) cmp, _, err := pushNotFilters(ctx, nil, node, nil, nil, nil) From 0f10cbaf5339714a93ea940471451c5c938a7c00 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 3 Oct 2024 12:21:00 -0700 Subject: [PATCH 34/39] feedback and fixing more tests --- server/handler.go | 5 +- sql/expression/function/queryinfo.go | 133 ++++++++++++++------------- sql/planbuilder/project.go | 34 +++---- sql/planbuilder/scalar.go | 5 + sql/query_flags.go | 7 ++ sql/rowexec/rel_iters.go | 4 + sql/rowexec/transaction_iters.go | 6 ++ 7 files changed, 108 insertions(+), 86 deletions(-) diff --git a/server/handler.go b/server/handler.go index f72336d276..24020d239a 100644 --- a/server/handler.go +++ b/server/handler.go @@ -425,7 +425,7 @@ func (h *Handler) doQuery( } }() - qFlags.Set(sql.QFlagDeferProjections) + qFlags.Set(sql.QFlagDeferProjections) // TODO: this somehow breaks timeout??????? schema, rowIter, qFlags, err := queryExec(sqlCtx, query, parsed, analyzedPlan, bindings, qFlags) if err != nil { sqlCtx.GetLogger().WithError(err).Warn("error running query") @@ -522,7 +522,7 @@ func GetDeferredProjections(iter sql.RowIter) []sql.Expression { if commitIter, isCommitIter := i.GetIter().(*rowexec.TransactionCommittingIter); isCommitIter { if projIter, isProjIter := commitIter.GetIter().(*rowexec.ProjectIter); isProjIter { if projIter.CanDefer() { - projIter.Defer() + commitIter.WithChildIter(projIter.GetChildIter()) return projIter.GetProjections() } } @@ -668,6 +668,7 @@ func (h *Handler) resultForDefaultIter( r.Rows = append(r.Rows, outputRow) r.RowsAffected++ case <-timer.C: + // TODO: timer should probably go in its own thread, as rowChan is blocking if h.readTimeout != 0 { // Cancel and return so Vitess can call the CloseConnection callback ctx.GetLogger().Tracef("connection timeout") diff --git a/sql/expression/function/queryinfo.go b/sql/expression/function/queryinfo.go index f53e80f836..902085ae93 100644 --- a/sql/expression/function/queryinfo.go +++ b/sql/expression/function/queryinfo.go @@ -27,104 +27,105 @@ import ( // RowCount implements the ROW_COUNT() function type RowCount struct{} -func (r RowCount) IsNonDeterministic() bool { - return true -} - func NewRowCount() sql.Expression { - return RowCount{} + return &RowCount{} } -var _ sql.FunctionExpression = RowCount{} -var _ sql.CollationCoercible = RowCount{} +var _ sql.FunctionExpression = &RowCount{} +var _ sql.CollationCoercible = &RowCount{} // Description implements sql.FunctionExpression -func (r RowCount) Description() string { +func (r *RowCount) Description() string { return "returns the number of rows updated." } // Resolved implements sql.Expression -func (r RowCount) Resolved() bool { +func (r *RowCount) Resolved() bool { return true } // String implements sql.Expression -func (r RowCount) String() string { +func (r *RowCount) String() string { return fmt.Sprintf("%s()", r.FunctionName()) } // Type implements sql.Expression -func (r RowCount) Type() sql.Type { +func (r *RowCount) Type() sql.Type { return types.Int64 } // CollationCoercibility implements the interface sql.CollationCoercible. -func (RowCount) CollationCoercibility(ctx *sql.Context) (collation sql.CollationID, coercibility byte) { +func (*RowCount) CollationCoercibility(ctx *sql.Context) (collation sql.CollationID, coercibility byte) { return sql.Collation_binary, 5 } // IsNullable implements sql.Expression -func (r RowCount) IsNullable() bool { +func (r *RowCount) IsNullable() bool { return false } // Eval implements sql.Expression -func (r RowCount) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { +func (r *RowCount) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return ctx.GetLastQueryInfoInt(sql.RowCount), nil } // Children implements sql.Expression -func (r RowCount) Children() []sql.Expression { +func (r *RowCount) Children() []sql.Expression { return nil } // WithChildren implements sql.Expression -func (r RowCount) WithChildren(children ...sql.Expression) (sql.Expression, error) { +func (r *RowCount) WithChildren(children ...sql.Expression) (sql.Expression, error) { return sql.NillaryWithChildren(r, children...) } // FunctionName implements sql.FunctionExpression -func (r RowCount) FunctionName() string { +func (r *RowCount) FunctionName() string { return "row_count" } +// IsNonDeterministic implements sql.NonDeterministicExpression +func (r *RowCount) IsNonDeterministic() bool { + return true +} + // LastInsertUuid implements the LAST_INSERT_UUID() function. This function is // NOT a standard function in MySQL, but is a useful analogue to LAST_INSERT_ID() // if customers are inserting UUIDs into a table. type LastInsertUuid struct{} -var _ sql.FunctionExpression = LastInsertUuid{} -var _ sql.CollationCoercible = LastInsertUuid{} +var _ sql.FunctionExpression = &LastInsertUuid{} +var _ sql.CollationCoercible = &LastInsertUuid{} func NewLastInsertUuid(children ...sql.Expression) (sql.Expression, error) { if len(children) > 0 { - return nil, sql.ErrInvalidChildrenNumber.New(LastInsertUuid{}.String(), len(children), 0) + return nil, sql.ErrInvalidChildrenNumber.New((&LastInsertUuid{}).String(), len(children), 0) } return &LastInsertUuid{}, nil } -func (l LastInsertUuid) CollationCoercibility(_ *sql.Context) (collation sql.CollationID, coercibility byte) { +func (l *LastInsertUuid) CollationCoercibility(_ *sql.Context) (collation sql.CollationID, coercibility byte) { return sql.Collation_binary, 5 } -func (l LastInsertUuid) Resolved() bool { +func (l *LastInsertUuid) Resolved() bool { return true } -func (l LastInsertUuid) String() string { +func (l *LastInsertUuid) String() string { return fmt.Sprintf("%s()", l.FunctionName()) } -func (l LastInsertUuid) Type() sql.Type { +func (l *LastInsertUuid) Type() sql.Type { return types.MustCreateStringWithDefaults(sqltypes.VarChar, 36) } -func (l LastInsertUuid) IsNullable() bool { +func (l *LastInsertUuid) IsNullable() bool { return false } -func (l LastInsertUuid) Eval(ctx *sql.Context, _ sql.Row) (interface{}, error) { +func (l *LastInsertUuid) Eval(ctx *sql.Context, _ sql.Row) (interface{}, error) { lastInsertUuid := ctx.GetLastQueryInfoString(sql.LastInsertUuid) result, _, err := l.Type().Convert(lastInsertUuid) if err != nil { @@ -133,19 +134,19 @@ func (l LastInsertUuid) Eval(ctx *sql.Context, _ sql.Row) (interface{}, error) { return result, nil } -func (l LastInsertUuid) Children() []sql.Expression { +func (l *LastInsertUuid) Children() []sql.Expression { return nil } -func (l LastInsertUuid) WithChildren(children ...sql.Expression) (sql.Expression, error) { +func (l *LastInsertUuid) WithChildren(children ...sql.Expression) (sql.Expression, error) { return NewLastInsertUuid(children...) } -func (l LastInsertUuid) FunctionName() string { +func (l *LastInsertUuid) FunctionName() string { return "last_insert_uuid" } -func (l LastInsertUuid) Description() string { +func (l *LastInsertUuid) Description() string { return "returns the first value of the UUID() function from the last INSERT statement." } @@ -155,56 +156,52 @@ type LastInsertId struct { expression.UnaryExpression } -func (r LastInsertId) IsNonDeterministic() bool { - return true -} - func NewLastInsertId(children ...sql.Expression) (sql.Expression, error) { switch len(children) { case 0: - return LastInsertId{}, nil + return &LastInsertId{}, nil case 1: - return LastInsertId{UnaryExpression: expression.UnaryExpression{Child: children[0]}}, nil + return &LastInsertId{UnaryExpression: expression.UnaryExpression{Child: children[0]}}, nil default: return nil, sql.ErrInvalidArgumentNumber.New("LastInsertId", len(children), 1) } } -var _ sql.FunctionExpression = LastInsertId{} -var _ sql.CollationCoercible = LastInsertId{} +var _ sql.FunctionExpression = &LastInsertId{} +var _ sql.CollationCoercible = &LastInsertId{} // Description implements sql.FunctionExpression -func (r LastInsertId) Description() string { +func (r *LastInsertId) Description() string { return "returns value of the AUTOINCREMENT column for the last INSERT." } // Resolved implements sql.Expression -func (r LastInsertId) Resolved() bool { +func (r *LastInsertId) Resolved() bool { return true } // String implements sql.Expression -func (r LastInsertId) String() string { +func (r *LastInsertId) String() string { return fmt.Sprintf("%s(%s)", r.FunctionName(), r.Child) } // Type implements sql.Expression -func (r LastInsertId) Type() sql.Type { +func (r *LastInsertId) Type() sql.Type { return types.Uint64 } // CollationCoercibility implements the interface sql.CollationCoercible. -func (LastInsertId) CollationCoercibility(ctx *sql.Context) (collation sql.CollationID, coercibility byte) { +func (*LastInsertId) CollationCoercibility(ctx *sql.Context) (collation sql.CollationID, coercibility byte) { return sql.Collation_binary, 5 } // IsNullable implements sql.Expression -func (r LastInsertId) IsNullable() bool { +func (r *LastInsertId) IsNullable() bool { return false } // Eval implements sql.Expression -func (r LastInsertId) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { +func (r *LastInsertId) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { // With no arguments, just return the last insert id for this session if len(r.Children()) == 0 { lastInsertId := ctx.GetLastQueryInfoInt(sql.LastInsertId) @@ -230,7 +227,7 @@ func (r LastInsertId) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { } // Children implements sql.Expression -func (r LastInsertId) Children() []sql.Expression { +func (r *LastInsertId) Children() []sql.Expression { if r.Child == nil { return nil } @@ -238,75 +235,81 @@ func (r LastInsertId) Children() []sql.Expression { } // WithChildren implements sql.Expression -func (r LastInsertId) WithChildren(children ...sql.Expression) (sql.Expression, error) { +func (r *LastInsertId) WithChildren(children ...sql.Expression) (sql.Expression, error) { return NewLastInsertId(children...) } // FunctionName implements sql.FunctionExpression -func (r LastInsertId) FunctionName() string { +func (r *LastInsertId) FunctionName() string { return "last_insert_id" } -// FoundRows implements the FOUND_ROWS() function -type FoundRows struct{} - -func (r FoundRows) IsNonDeterministic() bool { +// IsNonDeterministic implements sql.NonDeterministicExpression +func (r *LastInsertId) IsNonDeterministic() bool { return true } +// FoundRows implements the FOUND_ROWS() function +type FoundRows struct{} + func NewFoundRows() sql.Expression { - return FoundRows{} + return &FoundRows{} } -var _ sql.FunctionExpression = FoundRows{} -var _ sql.CollationCoercible = FoundRows{} +var _ sql.FunctionExpression = &FoundRows{} +var _ sql.CollationCoercible = &FoundRows{} // FunctionName implements sql.FunctionExpression -func (r FoundRows) FunctionName() string { +func (r *FoundRows) FunctionName() string { return "found_rows" } // Description implements sql.Expression -func (r FoundRows) Description() string { +func (r *FoundRows) Description() string { return "for a SELECT with a LIMIT clause, returns the number of rows that would be returned were there no LIMIT clause." } // Resolved implements sql.Expression -func (r FoundRows) Resolved() bool { +func (r *FoundRows) Resolved() bool { return true } // String implements sql.Expression -func (r FoundRows) String() string { +func (r *FoundRows) String() string { return fmt.Sprintf("%s()", r.FunctionName()) } // Type implements sql.Expression -func (r FoundRows) Type() sql.Type { +func (r *FoundRows) Type() sql.Type { return types.Int64 } // CollationCoercibility implements the interface sql.CollationCoercible. -func (FoundRows) CollationCoercibility(ctx *sql.Context) (collation sql.CollationID, coercibility byte) { +func (*FoundRows) CollationCoercibility(ctx *sql.Context) (collation sql.CollationID, coercibility byte) { return sql.Collation_binary, 5 } // IsNullable implements sql.Expression -func (r FoundRows) IsNullable() bool { +func (r *FoundRows) IsNullable() bool { return false } // Eval implements sql.Expression -func (r FoundRows) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { +func (r *FoundRows) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return ctx.GetLastQueryInfoInt(sql.FoundRows), nil } // Children implements sql.Expression -func (r FoundRows) Children() []sql.Expression { +func (r *FoundRows) Children() []sql.Expression { return nil } // WithChildren implements sql.Expression -func (r FoundRows) WithChildren(children ...sql.Expression) (sql.Expression, error) { +func (r *FoundRows) WithChildren(children ...sql.Expression) (sql.Expression, error) { return sql.NillaryWithChildren(r, children...) } + +// IsNonDeterministic implements sql.NonDeterministicExpression +func (r *FoundRows) IsNonDeterministic() bool { + return true +} \ No newline at end of file diff --git a/sql/planbuilder/project.go b/sql/planbuilder/project.go index 95d98f5a33..6897150b62 100644 --- a/sql/planbuilder/project.go +++ b/sql/planbuilder/project.go @@ -21,8 +21,7 @@ import ( "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/expression" - "github.com/dolthub/go-mysql-server/sql/expression/function" - "github.com/dolthub/go-mysql-server/sql/plan" + "github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/transform" ) @@ -195,16 +194,7 @@ func (b *Builder) selectExprToExpression(inScope *scope, se ast.SelectExpr) sql. return nil } -func (b *Builder) buildProjection(inScope, outScope *scope) { - projections := make([]sql.Expression, len(outScope.cols)) - for i, sc := range outScope.cols { - projections[i] = sc.scalar - } - proj, err := b.f.buildProject(plan.NewProject(projections, inScope.node), outScope.refsSubquery) - if err != nil { - b.handleErr(err) - } - outScope.node = proj +func (b *Builder) markDeferProjection(proj sql.Node, inScope, outScope *scope) { if !b.qFlags.IsSet(sql.QFlagDeferProjections) { return } @@ -214,16 +204,22 @@ func (b *Builder) buildProjection(inScope, outScope *scope) { if _, isProj := proj.(*plan.Project); !isProj { return } - for _, sc := range outScope.cols { - switch sc.scalar.(type) { - // TODO: column default expression are also not deferrable, but they don't appear in top level projections - case function.RowCount: - return - } - } proj.(*plan.Project).CanDefer = true } +func (b *Builder) buildProjection(inScope, outScope *scope) { + projections := make([]sql.Expression, len(outScope.cols)) + for i, sc := range outScope.cols { + projections[i] = sc.scalar + } + proj, err := b.f.buildProject(plan.NewProject(projections, inScope.node), outScope.refsSubquery) + if err != nil { + b.handleErr(err) + } + b.markDeferProjection(proj, inScope, outScope) + outScope.node = proj +} + func selectExprNeedsAlias(e *ast.AliasedExpr, expr sql.Expression) bool { if len(e.InputExpression) == 0 { return false diff --git a/sql/planbuilder/scalar.go b/sql/planbuilder/scalar.go index 55ff36f52b..87a52fe911 100644 --- a/sql/planbuilder/scalar.go +++ b/sql/planbuilder/scalar.go @@ -169,6 +169,11 @@ func (b *Builder) buildScalar(inScope *scope, e ast.Expr) (ex sql.Expression) { b.handleErr(err) } + switch rf.(type) { + case *function.Sleep, sql.NonDeterministicExpression: + b.qFlags.Unset(sql.QFlagDeferProjections) + } + // NOTE: Not all aggregate functions support DISTINCT. Fortunately, the vitess parser will throw // errors for when DISTINCT is used on aggregate functions that don't support DISTINCT. if v.Distinct { diff --git a/sql/query_flags.go b/sql/query_flags.go index 86c9310dff..432a1c8eef 100644 --- a/sql/query_flags.go +++ b/sql/query_flags.go @@ -59,6 +59,13 @@ func (qp *QueryFlags) Set(flag int) { qp.Flags.Add(flag) } +func (qp *QueryFlags) Unset(flag int) { + if qp == nil { + return + } + qp.Flags.Remove(flag) +} + func (qp *QueryFlags) IsSet(flag int) bool { return qp.Flags.Contains(flag) } diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index 7569306e19..b67b6cc5fe 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -158,6 +158,10 @@ func (i *ProjectIter) Defer() { i.deferred = true } +func (i *ProjectIter) GetChildIter() sql.RowIter { + return i.childIter +} + // ProjectRow evaluates a set of projections. func ProjectRow( ctx *sql.Context, diff --git a/sql/rowexec/transaction_iters.go b/sql/rowexec/transaction_iters.go index f358f96621..c60935175a 100644 --- a/sql/rowexec/transaction_iters.go +++ b/sql/rowexec/transaction_iters.go @@ -118,3 +118,9 @@ func (t *TransactionCommittingIter) Close(ctx *sql.Context) error { func (t *TransactionCommittingIter) GetIter() sql.RowIter { return t.childIter } + +func (t *TransactionCommittingIter) WithChildIter(childIter sql.RowIter) sql.RowIter { + nt := *t + t.childIter = childIter + return &nt +} From bd2a659dd7ff0cc50094130efaa91e34418eb579 Mon Sep 17 00:00:00 2001 From: jycor Date: Thu, 3 Oct 2024 19:22:19 +0000 Subject: [PATCH 35/39] [ga-format-pr] Run ./format_repo.sh to fix formatting --- sql/expression/function/queryinfo.go | 2 +- sql/planbuilder/project.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/expression/function/queryinfo.go b/sql/expression/function/queryinfo.go index 902085ae93..00174e8d75 100644 --- a/sql/expression/function/queryinfo.go +++ b/sql/expression/function/queryinfo.go @@ -312,4 +312,4 @@ func (r *FoundRows) WithChildren(children ...sql.Expression) (sql.Expression, er // IsNonDeterministic implements sql.NonDeterministicExpression func (r *FoundRows) IsNonDeterministic() bool { return true -} \ No newline at end of file +} diff --git a/sql/planbuilder/project.go b/sql/planbuilder/project.go index 6897150b62..1a24de5c2d 100644 --- a/sql/planbuilder/project.go +++ b/sql/planbuilder/project.go @@ -21,7 +21,7 @@ import ( "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/expression" - "github.com/dolthub/go-mysql-server/sql/plan" + "github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/transform" ) From a8b65defd8159a284aee5eafe0b882e7516b1e7f Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 3 Oct 2024 12:26:18 -0700 Subject: [PATCH 36/39] more tidying --- server/handler.go | 5 +++-- sql/rowexec/rel_iters.go | 8 -------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/server/handler.go b/server/handler.go index 24020d239a..c3e294a4ac 100644 --- a/server/handler.go +++ b/server/handler.go @@ -425,7 +425,7 @@ func (h *Handler) doQuery( } }() - qFlags.Set(sql.QFlagDeferProjections) // TODO: this somehow breaks timeout??????? + qFlags.Set(sql.QFlagDeferProjections) schema, rowIter, qFlags, err := queryExec(sqlCtx, query, parsed, analyzedPlan, bindings, qFlags) if err != nil { sqlCtx.GetLogger().WithError(err).Warn("error running query") @@ -513,7 +513,8 @@ func resultForEmptyIter(ctx *sql.Context, iter sql.RowIter, resultFields []*quer return &sqltypes.Result{Fields: resultFields}, nil } -// GetDeferredProjections looks for a top-level deferred projection, marks it as deferred, and retrieves its projections +// GetDeferredProjections looks for a top-level deferred projection, retrieves its projections, and removes it from the +// iterator tree. func GetDeferredProjections(iter sql.RowIter) []sql.Expression { switch i := iter.(type) { case *rowexec.ExprCloserIter: diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go index b67b6cc5fe..720c81018f 100644 --- a/sql/rowexec/rel_iters.go +++ b/sql/rowexec/rel_iters.go @@ -127,7 +127,6 @@ var _ sql.RowIter = &iters.JsonTableRowIter{} type ProjectIter struct { projs []sql.Expression canDefer bool - deferred bool childIter sql.RowIter } @@ -136,9 +135,6 @@ func (i *ProjectIter) Next(ctx *sql.Context) (sql.Row, error) { if err != nil { return nil, err } - if i.deferred { - return childRow, nil - } return ProjectRow(ctx, i.projs, childRow) } @@ -154,10 +150,6 @@ func (i *ProjectIter) CanDefer() bool { return i.canDefer } -func (i *ProjectIter) Defer() { - i.deferred = true -} - func (i *ProjectIter) GetChildIter() sql.RowIter { return i.childIter } From 988e247b217808bde557edb9554309fc288c3fac Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 3 Oct 2024 14:09:35 -0700 Subject: [PATCH 37/39] peek through limit iters as well --- server/handler.go | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/server/handler.go b/server/handler.go index c3e294a4ac..772d1bb8ec 100644 --- a/server/handler.go +++ b/server/handler.go @@ -40,6 +40,7 @@ import ( "github.com/dolthub/go-mysql-server/internal/sockstate" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/analyzer" + "github.com/dolthub/go-mysql-server/sql/iters" "github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/rowexec" "github.com/dolthub/go-mysql-server/sql/types" @@ -515,21 +516,33 @@ func resultForEmptyIter(ctx *sql.Context, iter sql.RowIter, resultFields []*quer // GetDeferredProjections looks for a top-level deferred projection, retrieves its projections, and removes it from the // iterator tree. -func GetDeferredProjections(iter sql.RowIter) []sql.Expression { +func GetDeferredProjections(iter sql.RowIter) (sql.RowIter, []sql.Expression) { switch i := iter.(type) { case *rowexec.ExprCloserIter: - return GetDeferredProjections(i.GetIter()) + _, projs := GetDeferredProjections(i.GetIter()) + return i, projs case *plan.TrackedRowIter: - if commitIter, isCommitIter := i.GetIter().(*rowexec.TransactionCommittingIter); isCommitIter { - if projIter, isProjIter := commitIter.GetIter().(*rowexec.ProjectIter); isProjIter { - if projIter.CanDefer() { - commitIter.WithChildIter(projIter.GetChildIter()) - return projIter.GetProjections() - } - } + _, projs := GetDeferredProjections(i.GetIter()) + return i, projs + case *rowexec.TransactionCommittingIter: + newChild, projs := GetDeferredProjections(i.GetIter()) + if projs != nil { + i.WithChildIter(newChild) + } + return i, projs + case *iters.LimitIter: + newChild, projs := GetDeferredProjections(i.ChildIter) + if projs != nil { + i.ChildIter = newChild } + return i, projs + case *rowexec.ProjectIter: + if i.CanDefer() { + return i.GetChildIter(), i.GetProjections() + } + return i, nil } - return nil + return iter, nil } // resultForMax1RowIter ensures that an empty iterator returns at most one row @@ -582,7 +595,7 @@ func (h *Handler) resultForDefaultIter( wg.Add(2) // Read rows off the row iterator and send them to the row channel. - projs := GetDeferredProjections(iter) + iter, projs := GetDeferredProjections(iter) var rowChan = make(chan sql.Row, 512) eg.Go(func() error { defer pan2err() From 4597dd04d90b29aaa41da832dd5389bf2846757b Mon Sep 17 00:00:00 2001 From: jycor Date: Thu, 3 Oct 2024 21:10:59 +0000 Subject: [PATCH 38/39] [ga-format-pr] Run ./format_repo.sh to fix formatting --- server/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/handler.go b/server/handler.go index 772d1bb8ec..09d0940e0a 100644 --- a/server/handler.go +++ b/server/handler.go @@ -527,7 +527,7 @@ func GetDeferredProjections(iter sql.RowIter) (sql.RowIter, []sql.Expression) { case *rowexec.TransactionCommittingIter: newChild, projs := GetDeferredProjections(i.GetIter()) if projs != nil { - i.WithChildIter(newChild) + i.WithChildIter(newChild) } return i, projs case *iters.LimitIter: From c30684ce2450aeca6b731fef3558c5312868c091 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 3 Oct 2024 16:22:55 -0700 Subject: [PATCH 39/39] another flag --- sql/planbuilder/project.go | 2 +- sql/planbuilder/scalar.go | 2 +- sql/query_flags.go | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/planbuilder/project.go b/sql/planbuilder/project.go index 1a24de5c2d..35a42c14a3 100644 --- a/sql/planbuilder/project.go +++ b/sql/planbuilder/project.go @@ -195,7 +195,7 @@ func (b *Builder) selectExprToExpression(inScope *scope, se ast.SelectExpr) sql. } func (b *Builder) markDeferProjection(proj sql.Node, inScope, outScope *scope) { - if !b.qFlags.IsSet(sql.QFlagDeferProjections) { + if !b.qFlags.IsSet(sql.QFlagDeferProjections) || b.qFlags.IsSet(sql.QFlagUndeferrableExprs) { return } if inScope.parent != nil && inScope.parent.activeSubquery != nil { diff --git a/sql/planbuilder/scalar.go b/sql/planbuilder/scalar.go index 87a52fe911..2ee0ec8c78 100644 --- a/sql/planbuilder/scalar.go +++ b/sql/planbuilder/scalar.go @@ -171,7 +171,7 @@ func (b *Builder) buildScalar(inScope *scope, e ast.Expr) (ex sql.Expression) { switch rf.(type) { case *function.Sleep, sql.NonDeterministicExpression: - b.qFlags.Unset(sql.QFlagDeferProjections) + b.qFlags.Set(sql.QFlagUndeferrableExprs) } // NOTE: Not all aggregate functions support DISTINCT. Fortunately, the vitess parser will throw diff --git a/sql/query_flags.go b/sql/query_flags.go index 432a1c8eef..fcdd053c31 100644 --- a/sql/query_flags.go +++ b/sql/query_flags.go @@ -46,6 +46,8 @@ const ( // QFlagDeferProjections indicates that a top-level projections for this query should be deferred and handled by // RowToSQL QFlagDeferProjections + // QFlagUndeferrableExprs indicates that the query has expressions that cannot be deferred + QFlagUndeferrableExprs ) type QueryFlags struct {