You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

476 lines
16 KiB

// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package planner
import (
"context"
"math"
"runtime/trace"
"strings"
"time"
"github.com/pingcap/errors"
"github.com/pingcap/parser"
"github.com/pingcap/parser/ast"
"github.com/pingcap/tidb/bindinfo"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/infoschema"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/planner/cascades"
plannercore "github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/privilege"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/stmtctx"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/hint"
"github.com/pingcap/tidb/util/logutil"
"go.uber.org/zap"
)
// GetPreparedStmt extract the prepared statement from the execute statement.
func GetPreparedStmt(stmt *ast.ExecuteStmt, vars *variable.SessionVars) (ast.StmtNode, error) {
var ok bool
execID := stmt.ExecID
if stmt.Name != "" {
if execID, ok = vars.PreparedStmtNameToID[stmt.Name]; !ok {
return nil, plannercore.ErrStmtNotFound
}
}
if preparedPointer, ok := vars.PreparedStmts[execID]; ok {
preparedObj, ok := preparedPointer.(*plannercore.CachedPrepareStmt)
if !ok {
return nil, errors.Errorf("invalid CachedPrepareStmt type")
}
return preparedObj.PreparedAst.Stmt, nil
}
return nil, plannercore.ErrStmtNotFound
}
// IsReadOnly check whether the ast.Node is a read only statement.
func IsReadOnly(node ast.Node, vars *variable.SessionVars) bool {
if execStmt, isExecStmt := node.(*ast.ExecuteStmt); isExecStmt {
s, err := GetPreparedStmt(execStmt, vars)
if err != nil {
logutil.BgLogger().Warn("GetPreparedStmt failed", zap.Error(err))
return false
}
return ast.IsReadOnly(s)
}
return ast.IsReadOnly(node)
}
// Optimize does optimization and creates a Plan.
// The node must be prepared first.
func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is infoschema.InfoSchema) (plannercore.Plan, types.NameSlice, error) {
sessVars := sctx.GetSessionVars()
// Because for write stmt, TiFlash has a different results when lock the data in point get plan. We ban the TiFlash
// engine in not read only stmt.
if _, isolationReadContainTiFlash := sessVars.IsolationReadEngines[kv.TiFlash]; isolationReadContainTiFlash && !IsReadOnly(node, sessVars) {
delete(sessVars.IsolationReadEngines, kv.TiFlash)
defer func() {
sessVars.IsolationReadEngines[kv.TiFlash] = struct{}{}
}()
}
if _, isolationReadContainTiKV := sctx.GetSessionVars().GetIsolationReadEngines()[kv.TiKV]; isolationReadContainTiKV {
fp := plannercore.TryFastPlan(sctx, node)
if fp != nil {
if !useMaxTS(sctx, fp) {
sctx.PrepareTSFuture(ctx)
}
return fp, fp.OutputNames(), nil
}
}
sctx.PrepareTSFuture(ctx)
tableHints := hint.ExtractTableHintsFromStmtNode(node, sctx)
stmtHints, warns := handleStmtHints(tableHints)
sessVars.StmtCtx.StmtHints = stmtHints
for _, warn := range warns {
sctx.GetSessionVars().StmtCtx.AppendWarning(warn)
}
warns = warns[:0]
bestPlan, names, _, err := optimize(ctx, sctx, node, is)
if err != nil {
return nil, nil, err
}
if !(sessVars.UsePlanBaselines || sessVars.EvolvePlanBaselines) {
return bestPlan, names, nil
}
stmtNode, ok := node.(ast.StmtNode)
if !ok {
return bestPlan, names, nil
}
bindRecord, scope := getBindRecord(sctx, stmtNode)
if bindRecord == nil {
return bestPlan, names, nil
}
if sctx.GetSessionVars().SelectLimit != math.MaxUint64 {
sctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("sql_select_limit is set, so plan binding is not activated"))
return bestPlan, names, nil
}
bestPlanHint := plannercore.GenHintsFromPhysicalPlan(bestPlan)
if len(bindRecord.Bindings) > 0 {
orgBinding := bindRecord.Bindings[0] // the first is the original binding
for _, tbHint := range tableHints { // consider table hints which contained by the original binding
if orgBinding.Hint.ContainTableHint(tbHint.HintName.String()) {
bestPlanHint = append(bestPlanHint, tbHint)
}
}
}
bestPlanHintStr := hint.RestoreOptimizerHints(bestPlanHint)
defer func() {
sessVars.StmtCtx.StmtHints = stmtHints
for _, warn := range warns {
sctx.GetSessionVars().StmtCtx.AppendWarning(warn)
}
}()
binding := bindRecord.FindBinding(bestPlanHintStr)
// If the best bestPlan is in baselines, just use it.
if binding != nil && binding.Status == bindinfo.Using {
if sctx.GetSessionVars().UsePlanBaselines {
stmtHints, warns = handleStmtHints(binding.Hint.GetFirstTableHints())
}
return bestPlan, names, nil
}
bestCostAmongHints := math.MaxFloat64
var bestPlanAmongHints plannercore.Plan
originHints := hint.CollectHint(stmtNode)
// Try to find the best binding.
for _, binding := range bindRecord.Bindings {
if binding.Status != bindinfo.Using {
continue
}
metrics.BindUsageCounter.WithLabelValues(scope).Inc()
hint.BindHint(stmtNode, binding.Hint)
curStmtHints, curWarns := handleStmtHints(binding.Hint.GetFirstTableHints())
sctx.GetSessionVars().StmtCtx.StmtHints = curStmtHints
plan, _, cost, err := optimize(ctx, sctx, node, is)
if err != nil {
binding.Status = bindinfo.Invalid
handleInvalidBindRecord(ctx, sctx, scope, bindinfo.BindRecord{
OriginalSQL: bindRecord.OriginalSQL,
Db: bindRecord.Db,
Bindings: []bindinfo.Binding{binding},
})
continue
}
if cost < bestCostAmongHints {
if sctx.GetSessionVars().UsePlanBaselines {
stmtHints, warns = curStmtHints, curWarns
}
bestCostAmongHints = cost
bestPlanAmongHints = plan
}
}
// 1. If it is a select query.
// 2. If there is already a evolution task, we do not need to handle it again.
// 3. If the origin binding contain `read_from_storage` hint, we should ignore the evolve task.
// 4. If the best plan contain TiFlash hint, we should ignore the evolve task.
if _, ok := stmtNode.(*ast.SelectStmt); ok &&
sctx.GetSessionVars().EvolvePlanBaselines && binding == nil &&
!originHints.ContainTableHint(plannercore.HintReadFromStorage) &&
!bindRecord.Bindings[0].Hint.ContainTableHint(plannercore.HintReadFromStorage) {
handleEvolveTasks(ctx, sctx, bindRecord, stmtNode, bestPlanHintStr)
}
// Restore the hint to avoid changing the stmt node.
hint.BindHint(stmtNode, originHints)
if sctx.GetSessionVars().UsePlanBaselines && bestPlanAmongHints != nil {
return bestPlanAmongHints, names, nil
}
return bestPlan, names, nil
}
func optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is infoschema.InfoSchema) (plannercore.Plan, types.NameSlice, float64, error) {
// build logical plan
sctx.GetSessionVars().PlanID = 0
sctx.GetSessionVars().PlanColumnID = 0
hintProcessor := &hint.BlockHintProcessor{Ctx: sctx}
node.Accept(hintProcessor)
builder := plannercore.NewPlanBuilder(sctx, is, hintProcessor)
// reset fields about rewrite
sctx.GetSessionVars().RewritePhaseInfo.Reset()
beginRewrite := time.Now()
p, err := builder.Build(ctx, node)
if err != nil {
return nil, nil, 0, err
}
sctx.GetSessionVars().RewritePhaseInfo.DurationRewrite = time.Since(beginRewrite)
sctx.GetSessionVars().StmtCtx.Tables = builder.GetDBTableInfo()
activeRoles := sctx.GetSessionVars().ActiveRoles
// Check privilege. Maybe it's better to move this to the Preprocess, but
// we need the table information to check privilege, which is collected
// into the visitInfo in the logical plan builder.
if pm := privilege.GetPrivilegeManager(sctx); pm != nil {
if err := plannercore.CheckPrivilege(activeRoles, pm, builder.GetVisitInfo()); err != nil {
return nil, nil, 0, err
}
}
if err := plannercore.CheckTableLock(sctx, is, builder.GetVisitInfo()); err != nil {
return nil, nil, 0, err
}
// Handle the execute statement.
if execPlan, ok := p.(*plannercore.Execute); ok {
err := execPlan.OptimizePreparedPlan(ctx, sctx, is)
return p, p.OutputNames(), 0, err
}
names := p.OutputNames()
// Handle the non-logical plan statement.
logic, isLogicalPlan := p.(plannercore.LogicalPlan)
if !isLogicalPlan {
return p, names, 0, nil
}
// Handle the logical plan statement, use cascades planner if enabled.
if sctx.GetSessionVars().GetEnableCascadesPlanner() {
finalPlan, cost, err := cascades.DefaultOptimizer.FindBestPlan(sctx, logic)
return finalPlan, names, cost, err
}
finalPlan, cost, err := plannercore.DoOptimize(ctx, sctx, builder.GetOptFlag(), logic)
return finalPlan, names, cost, err
}
func extractSelectAndNormalizeDigest(stmtNode ast.StmtNode) (ast.StmtNode, string, string) {
switch x := stmtNode.(type) {
case *ast.ExplainStmt:
switch x.Stmt.(type) {
case *ast.SelectStmt, *ast.DeleteStmt, *ast.UpdateStmt, *ast.InsertStmt:
plannercore.EraseLastSemicolon(x)
normalizeExplainSQL := parser.Normalize(x.Text())
idx := int(0)
switch n := x.Stmt.(type) {
case *ast.SelectStmt:
idx = strings.Index(normalizeExplainSQL, "select")
case *ast.DeleteStmt:
idx = strings.Index(normalizeExplainSQL, "delete")
case *ast.UpdateStmt:
idx = strings.Index(normalizeExplainSQL, "update")
case *ast.InsertStmt:
if n.IsReplace {
idx = strings.Index(normalizeExplainSQL, "replace")
} else {
idx = strings.Index(normalizeExplainSQL, "insert")
}
}
normalizeSQL := normalizeExplainSQL[idx:]
hash := parser.DigestNormalized(normalizeSQL)
return x.Stmt, normalizeSQL, hash
}
case *ast.SelectStmt, *ast.DeleteStmt, *ast.UpdateStmt, *ast.InsertStmt:
plannercore.EraseLastSemicolon(x)
normalizedSQL, hash := parser.NormalizeDigest(x.Text())
return x, normalizedSQL, hash
}
return nil, "", ""
}
func getBindRecord(ctx sessionctx.Context, stmt ast.StmtNode) (*bindinfo.BindRecord, string) {
// When the domain is initializing, the bind will be nil.
if ctx.Value(bindinfo.SessionBindInfoKeyType) == nil {
return nil, ""
}
selectStmt, normalizedSQL, hash := extractSelectAndNormalizeDigest(stmt)
if selectStmt == nil {
return nil, ""
}
sessionHandle := ctx.Value(bindinfo.SessionBindInfoKeyType).(*bindinfo.SessionHandle)
bindRecord := sessionHandle.GetBindRecord(normalizedSQL, ctx.GetSessionVars().CurrentDB)
if bindRecord == nil {
bindRecord = sessionHandle.GetBindRecord(normalizedSQL, "")
}
if bindRecord != nil {
if bindRecord.HasUsingBinding() {
return bindRecord, metrics.ScopeSession
}
return nil, ""
}
globalHandle := domain.GetDomain(ctx).BindHandle()
if globalHandle == nil {
return nil, ""
}
bindRecord = globalHandle.GetBindRecord(hash, normalizedSQL, ctx.GetSessionVars().CurrentDB)
if bindRecord == nil {
bindRecord = globalHandle.GetBindRecord(hash, normalizedSQL, "")
}
return bindRecord, metrics.ScopeGlobal
}
func handleInvalidBindRecord(ctx context.Context, sctx sessionctx.Context, level string, bindRecord bindinfo.BindRecord) {
sessionHandle := sctx.Value(bindinfo.SessionBindInfoKeyType).(*bindinfo.SessionHandle)
err := sessionHandle.DropBindRecord(bindRecord.OriginalSQL, bindRecord.Db, &bindRecord.Bindings[0])
if err != nil {
logutil.Logger(ctx).Info("drop session bindings failed")
}
if level == metrics.ScopeSession {
return
}
globalHandle := domain.GetDomain(sctx).BindHandle()
globalHandle.AddDropInvalidBindTask(&bindRecord)
}
func handleEvolveTasks(ctx context.Context, sctx sessionctx.Context, br *bindinfo.BindRecord, stmtNode ast.StmtNode, planHint string) {
bindSQL := bindinfo.GenerateBindSQL(ctx, stmtNode, planHint)
if bindSQL == "" {
return
}
charset, collation := sctx.GetSessionVars().GetCharsetInfo()
binding := bindinfo.Binding{
BindSQL: bindSQL,
Status: bindinfo.PendingVerify,
Charset: charset,
Collation: collation,
Source: bindinfo.Evolve,
}
globalHandle := domain.GetDomain(sctx).BindHandle()
globalHandle.AddEvolvePlanTask(br.OriginalSQL, br.Db, binding)
}
// useMaxTS returns true when meets following conditions:
// 1. ctx is auto commit tagged.
// 2. plan is point get by pk.
func useMaxTS(ctx sessionctx.Context, p plannercore.Plan) bool {
if !plannercore.IsAutoCommitTxn(ctx) {
return false
}
v, ok := p.(*plannercore.PointGetPlan)
return ok && v.IndexInfo == nil
}
// OptimizeExecStmt to optimize prepare statement protocol "execute" statement
// this is a short path ONLY does things filling prepare related params
// for point select like plan which does not need extra things
func OptimizeExecStmt(ctx context.Context, sctx sessionctx.Context,
execAst *ast.ExecuteStmt, is infoschema.InfoSchema) (plannercore.Plan, error) {
defer trace.StartRegion(ctx, "Optimize").End()
var err error
builder := plannercore.NewPlanBuilder(sctx, is, nil)
p, err := builder.Build(ctx, execAst)
if err != nil {
return nil, err
}
if execPlan, ok := p.(*plannercore.Execute); ok {
err = execPlan.OptimizePreparedPlan(ctx, sctx, is)
return execPlan.Plan, err
}
err = errors.Errorf("invalid result plan type, should be Execute")
return nil, err
}
func handleStmtHints(hints []*ast.TableOptimizerHint) (stmtHints stmtctx.StmtHints, warns []error) {
if len(hints) == 0 {
return
}
var memoryQuotaHint, useToJAHint, useCascadesHint, maxExecutionTime *ast.TableOptimizerHint
var memoryQuotaHintCnt, useToJAHintCnt, useCascadesHintCnt, noIndexMergeHintCnt, readReplicaHintCnt, maxExecutionTimeCnt int
for _, hint := range hints {
switch hint.HintName.L {
case "memory_quota":
memoryQuotaHint = hint
memoryQuotaHintCnt++
case "use_toja":
useToJAHint = hint
useToJAHintCnt++
case "use_cascades":
useCascadesHint = hint
useCascadesHintCnt++
case "no_index_merge":
noIndexMergeHintCnt++
case "read_consistent_replica":
readReplicaHintCnt++
case "max_execution_time":
maxExecutionTimeCnt++
maxExecutionTime = hint
}
}
// Handle MEMORY_QUOTA
if memoryQuotaHintCnt != 0 {
if memoryQuotaHintCnt > 1 {
warn := errors.New("There are multiple MEMORY_QUOTA hints, only the last one will take effect")
warns = append(warns, warn)
}
// Executor use MemoryQuota <= 0 to indicate no memory limit, here use < 0 to handle hint syntax error.
if memoryQuota := memoryQuotaHint.HintData.(int64); memoryQuota < 0 {
warn := errors.New("The use of MEMORY_QUOTA hint is invalid, valid usage: MEMORY_QUOTA(10 MB) or MEMORY_QUOTA(10 GB)")
warns = append(warns, warn)
} else {
stmtHints.HasMemQuotaHint = true
stmtHints.MemQuotaQuery = memoryQuota
if memoryQuota == 0 {
warn := errors.New("Setting the MEMORY_QUOTA to 0 means no memory limit")
warns = append(warns, warn)
}
}
}
// Handle USE_TOJA
if useToJAHintCnt != 0 {
if useToJAHintCnt > 1 {
warn := errors.New("There are multiple USE_TOJA hints, only the last one will take effect")
warns = append(warns, warn)
}
stmtHints.HasAllowInSubqToJoinAndAggHint = true
stmtHints.AllowInSubqToJoinAndAgg = useToJAHint.HintData.(bool)
}
// Handle USE_CASCADES
if useCascadesHintCnt != 0 {
if useCascadesHintCnt > 1 {
warn := errors.Errorf("USE_CASCADES() is defined more than once, only the last definition takes effect: USE_CASCADES(%v)", useCascadesHint.HintData.(bool))
warns = append(warns, warn)
}
stmtHints.HasEnableCascadesPlannerHint = true
stmtHints.EnableCascadesPlanner = useCascadesHint.HintData.(bool)
}
// Handle NO_INDEX_MERGE
if noIndexMergeHintCnt != 0 {
if noIndexMergeHintCnt > 1 {
warn := errors.New("There are multiple NO_INDEX_MERGE hints, only the last one will take effect")
warns = append(warns, warn)
}
stmtHints.NoIndexMergeHint = true
}
// Handle READ_CONSISTENT_REPLICA
if readReplicaHintCnt != 0 {
if readReplicaHintCnt > 1 {
warn := errors.New("There are multiple READ_CONSISTENT_REPLICA hints, only the last one will take effect")
warns = append(warns, warn)
}
stmtHints.HasReplicaReadHint = true
stmtHints.ReplicaRead = byte(kv.ReplicaReadFollower)
}
// Handle MAX_EXECUTION_TIME
if maxExecutionTimeCnt != 0 {
if maxExecutionTimeCnt > 1 {
warn := errors.New("There are multiple MAX_EXECUTION_TIME hints, only the last one will take effect")
warns = append(warns, warn)
}
stmtHints.HasMaxExecutionTime = true
stmtHints.MaxExecutionTime = maxExecutionTime.HintData.(uint64)
}
return
}
func init() {
plannercore.OptimizeAstNode = Optimize
}