// Copyright 2020 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // See the License for the specific language governing permissions and // limitations under the License. package executor import ( "context" "fmt" "strings" "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/infoschema" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/sqlexec" ) type inspectionSummaryRetriever struct { dummyCloser retrieved bool table *model.TableInfo extractor *plannercore.InspectionSummaryTableExtractor timeRange plannercore.QueryTimeRange } // inspectionSummaryRules is used to maintain var inspectionSummaryRules = map[string][]string{ "query-summary": { "tidb_connection_count", "tidb_query_duration", "tidb_qps_ideal", "tidb_qps", "tidb_ops_internal", "tidb_ops_statement", "tidb_failed_query_opm", "tidb_slow_query_duration", "tidb_slow_query_cop_wait_duration", "tidb_slow_query_cop_process_duration", }, "wait-events": { "tidb_get_token_duration", "tidb_load_schema_duration", "tidb_query_duration", "tidb_parse_duration", "tidb_compile_duration", "tidb_execute_duration", "tidb_auto_id_request_duration", "pd_tso_wait_duration", "pd_tso_rpc_duration", "tidb_distsql_execution_duration", "pd_start_tso_wait_duration", "tidb_transaction_local_latch_wait_duration", "tidb_transaction_duration", "pd_request_rpc_duration", "tidb_cop_duration", "tidb_batch_client_wait_duration", "tidb_batch_client_unavailable_duration", "tidb_kv_backoff_duration", "tidb_kv_request_duration", "pd_client_cmd_duration", "tikv_grpc_message_duration", "tikv_average_grpc_messge_duration", "tikv_channel_full", "tikv_scheduler_is_busy", "tikv_coprocessor_is_busy", "tikv_engine_write_stall", "tikv_raftstore_apply_log_avg_duration", "tikv_raftstore_apply_log_duration", "tikv_raftstore_append_log_avg_duration", "tikv_raftstore_append_log_duration", "tikv_raftstore_commit_log_avg_duration", "tikv_raftstore_commit_log_duration", "tikv_raftstore_process_duration", "tikv_raftstore_propose_wait_duration", "tikv_propose_avg_wait_duration", "tikv_raftstore_apply_wait_duration", "tikv_apply_avg_wait_duration", "tikv_check_split_duration", "tikv_storage_async_request_duration", "tikv_storage_async_request_avg_duration", "tikv_scheduler_command_duration", "tikv_scheduler_command_avg_duration", "tikv_scheduler_latch_wait_duration", "tikv_scheduler_latch_wait_avg_duration", "tikv_send_snapshot_duration", "tikv_handle_snapshot_duration", "tikv_cop_request_durations", "tikv_cop_request_duration", "tikv_cop_handle_duration", "tikv_cop_wait_duration", "tikv_engine_max_get_duration", "tikv_engine_avg_get_duration", "tikv_engine_avg_seek_duration", "tikv_engine_write_duration", "tikv_wal_sync_max_duration", "tikv_wal_sync_duration", "tikv_compaction_max_duration", "tikv_compaction_duration", "tikv_sst_read_max_duration", "tikv_sst_read_duration", "tikv_write_stall_max_duration", "tikv_write_stall_avg_duration", "tikv_oldest_snapshots_duration", "tikv_ingest_sst_duration", "tikv_ingest_sst_avg_duration", "tikv_engine_blob_seek_duration", "tikv_engine_blob_get_duration", "tikv_engine_blob_file_read_duration", "tikv_engine_blob_file_write_duration", "tikv_engine_blob_file_sync_duration", "tikv_lock_manager_waiter_lifetime_avg_duration", "tikv_lock_manager_deadlock_detect_duration", "tikv_lock_manager_deadlock_detect_avg_duration", }, "read-link": { "tidb_get_token_duration", "tidb_parse_duration", "tidb_compile_duration", "pd_tso_rpc_duration", "pd_tso_wait_duration", "tidb_execute_duration", "tidb_expensive_executors_ops", "tidb_query_using_plan_cache_ops", "tidb_distsql_execution_duration", "tidb_distsql_partial_num", "tidb_distsql_partial_qps", "tidb_distsql_partial_scan_key_num", "tidb_distsql_qps", "tidb_distsql_scan_key_num", "tidb_distsql_copr_cache", "tidb_region_cache_ops", "tidb_batch_client_pending_req_count", "tidb_batch_client_unavailable_duration", "tidb_batch_client_wait_duration", "tidb_kv_backoff_duration", "tidb_kv_backoff_ops", "tidb_kv_region_error_ops", "tidb_kv_request_duration", "tidb_kv_request_ops", "tidb_kv_snapshot_ops", "tidb_kv_txn_ops", "tikv_average_grpc_messge_duration", "tikv_grpc_avg_req_batch_size", "tikv_grpc_avg_resp_batch_size", "tikv_grpc_errors", "tikv_grpc_message_duration", "tikv_grpc_qps", "tikv_grpc_req_batch_size", "tikv_grpc_resp_batch_size", "tidb_cop_duration", "tikv_cop_wait_duration", "tikv_coprocessor_is_busy", "tikv_coprocessor_request_error", "tikv_cop_handle_duration", "tikv_cop_kv_cursor_operations", "tikv_cop_request_duration", "tikv_cop_request_durations", "tikv_cop_scan_details", "tikv_cop_dag_executors_ops", "tikv_cop_dag_requests_ops", "tikv_cop_scan_keys_num", "tikv_cop_requests_ops", "tikv_cop_total_response_size_per_seconds", "tikv_cop_total_rocksdb_perf_statistics", "tikv_channel_full", "tikv_engine_avg_get_duration", "tikv_engine_avg_seek_duration", "tikv_handle_snapshot_duration", "tikv_block_all_cache_hit", "tikv_block_bloom_prefix_cache_hit", "tikv_block_cache_size", "tikv_block_data_cache_hit", "tikv_block_filter_cache_hit", "tikv_block_index_cache_hit", "tikv_engine_get_block_cache_operations", "tikv_engine_get_cpu_cache_operations", "tikv_engine_get_memtable_operations", "tikv_per_read_avg_bytes", "tikv_per_read_max_bytes", }, "write-link": { "tidb_get_token_duration", "tidb_parse_duration", "tidb_compile_duration", "pd_tso_rpc_duration", "pd_tso_wait_duration", "tidb_execute_duration", "tidb_transaction_duration", "tidb_transaction_local_latch_wait_duration", "tidb_transaction_ops", "tidb_transaction_retry_error_ops", "tidb_transaction_retry_num", "tidb_transaction_statement_num", "tidb_auto_id_qps", "tidb_auto_id_request_duration", "tidb_region_cache_ops", "tidb_kv_backoff_duration", "tidb_kv_backoff_ops", "tidb_kv_region_error_ops", "tidb_kv_request_duration", "tidb_kv_request_ops", "tidb_kv_snapshot_ops", "tidb_kv_txn_ops", "tidb_kv_write_num", "tidb_kv_write_size", "tikv_average_grpc_messge_duration", "tikv_grpc_avg_req_batch_size", "tikv_grpc_avg_resp_batch_size", "tikv_grpc_errors", "tikv_grpc_message_duration", "tikv_grpc_qps", "tikv_grpc_req_batch_size", "tikv_grpc_resp_batch_size", "tikv_scheduler_command_avg_duration", "tikv_scheduler_command_duration", "tikv_scheduler_is_busy", "tikv_scheduler_keys_read_avg", "tikv_scheduler_keys_read", "tikv_scheduler_keys_written_avg", "tikv_scheduler_keys_written", "tikv_scheduler_latch_wait_avg_duration", "tikv_scheduler_latch_wait_duration", "tikv_scheduler_pending_commands", "tikv_scheduler_priority_commands", "tikv_scheduler_scan_details", "tikv_scheduler_stage", "tikv_scheduler_writing_bytes", "tikv_propose_avg_wait_duration", "tikv_raftstore_propose_wait_duration", "tikv_raftstore_append_log_avg_duration", "tikv_raftstore_append_log_duration", "tikv_raftstore_commit_log_avg_duration", "tikv_raftstore_commit_log_duration", "tikv_apply_avg_wait_duration", "tikv_raftstore_apply_log_avg_duration", "tikv_raftstore_apply_log_duration", "tikv_raftstore_apply_wait_duration", "tikv_engine_wal_sync_operations", "tikv_engine_write_duration", "tikv_engine_write_operations", "tikv_engine_write_stall", "tikv_write_stall_avg_duration", "tikv_write_stall_max_duration", "tikv_write_stall_reason", }, "ddl": { "tidb_ddl_add_index_speed", "tidb_ddl_batch_add_index_duration", "tidb_ddl_deploy_syncer_duration", "tidb_ddl_duration", "tidb_ddl_meta_opm", "tidb_ddl_opm", "tidb_ddl_update_self_version_duration", "tidb_ddl_waiting_jobs_num", "tidb_ddl_worker_duration", }, "stats": { "tidb_statistics_auto_analyze_duration", "tidb_statistics_auto_analyze_ops", "tidb_statistics_dump_feedback_ops", "tidb_statistics_fast_analyze_status", "tidb_statistics_pseudo_estimation_ops", "tidb_statistics_significant_feedback", "tidb_statistics_stats_inaccuracy_rate", "tidb_statistics_store_query_feedback_qps", "tidb_statistics_update_stats_ops", }, "gc": { "tidb_gc_action_result_opm", "tidb_gc_config", "tidb_gc_delete_range_fail_opm", "tidb_gc_delete_range_task_status", "tidb_gc_duration", "tidb_gc_fail_opm", "tidb_gc_push_task_duration", "tidb_gc_too_many_locks_opm", "tidb_gc_worker_action_opm", "tikv_engine_blob_gc_duration", "tikv_auto_gc_progress", "tikv_auto_gc_safepoint", "tikv_auto_gc_working", "tikv_gc_fail_tasks", "tikv_gc_keys", "tikv_gc_skipped_tasks", "tikv_gc_speed", "tikv_gc_tasks_avg_duration", "tikv_gc_tasks_duration", "tikv_gc_too_busy", "tikv_gc_tasks_ops", }, "rocksdb": { "tikv_compaction_duration", "tikv_compaction_max_duration", "tikv_compaction_operations", "tikv_compaction_pending_bytes", "tikv_compaction_reason", "tikv_write_stall_avg_duration", "tikv_write_stall_max_duration", "tikv_write_stall_reason", "store_available_ratio", "store_size_amplification", "tikv_engine_avg_get_duration", "tikv_engine_avg_seek_duration", "tikv_engine_blob_bytes_flow", "tikv_engine_blob_file_count", "tikv_engine_blob_file_read_duration", "tikv_engine_blob_file_size", "tikv_engine_blob_file_sync_duration", "tikv_engine_blob_file_sync_operations", "tikv_engine_blob_file_write_duration", "tikv_engine_blob_gc_bytes_flow", "tikv_engine_blob_gc_duration", "tikv_engine_blob_gc_file", "tikv_engine_blob_gc_keys_flow", "tikv_engine_blob_get_duration", "tikv_engine_blob_key_avg_size", "tikv_engine_blob_key_max_size", "tikv_engine_blob_seek_duration", "tikv_engine_blob_seek_operations", "tikv_engine_blob_value_avg_size", "tikv_engine_blob_value_max_size", "tikv_engine_compaction_flow_bytes", "tikv_engine_get_block_cache_operations", "tikv_engine_get_cpu_cache_operations", "tikv_engine_get_memtable_operations", "tikv_engine_live_blob_size", "tikv_engine_max_get_duration", "tikv_engine_max_seek_duration", "tikv_engine_seek_operations", "tikv_engine_size", "tikv_engine_wal_sync_operations", "tikv_engine_write_duration", "tikv_engine_write_operations", "tikv_engine_write_stall", }, "pd": { "pd_scheduler_balance_region", "pd_balance_scheduler_status", "pd_checker_event_count", "pd_client_cmd_duration", "pd_client_cmd_ops", "pd_cluster_metadata", "pd_cluster_status", "pd_grpc_completed_commands_duration", "pd_grpc_completed_commands_rate", "pd_request_rpc_duration", "pd_request_rpc_ops", "pd_request_rpc_duration_avg", "pd_handle_transactions_duration", "pd_handle_transactions_rate", "pd_hotspot_status", "pd_label_distribution", "pd_operator_finish_duration", "pd_operator_step_finish_duration", "pd_peer_round_trip_duration", "pd_region_health", "pd_region_heartbeat_duration", "pd_region_label_isolation_level", "pd_region_syncer_status", "pd_role", "pd_schedule_filter", "pd_schedule_operator", "pd_schedule_store_limit", "pd_scheduler_balance_direction", "pd_scheduler_balance_leader", "pd_scheduler_config", "pd_scheduler_op_influence", "pd_scheduler_region_heartbeat", "pd_scheduler_status", "pd_scheduler_store_status", "pd_scheduler_tolerant_resource", "pd_server_etcd_state", "pd_start_tso_wait_duration", }, "raftstore": { "tikv_approximate_avg_region_size", "tikv_approximate_region_size_histogram", "tikv_approximate_region_size", "tikv_raftstore_append_log_avg_duration", "tikv_raftstore_append_log_duration", "tikv_raftstore_commit_log_avg_duration", "tikv_raftstore_commit_log_duration", "tikv_apply_avg_wait_duration", "tikv_raftstore_apply_log_avg_duration", "tikv_raftstore_apply_log_duration", "tikv_raftstore_apply_wait_duration", "tikv_raftstore_process_duration", "tikv_raftstore_process_handled", "tikv_propose_avg_wait_duration", "tikv_raftstore_propose_wait_duration", "tikv_raft_dropped_messages", "tikv_raft_log_speed", "tikv_raft_message_avg_batch_size", "tikv_raft_message_batch_size", "tikv_raft_proposals_per_ready", "tikv_raft_proposals", "tikv_raft_sent_messages", }, } func (e *inspectionSummaryRetriever) retrieve(ctx context.Context, sctx sessionctx.Context) ([][]types.Datum, error) { if e.retrieved || e.extractor.SkipInspection { return nil, nil } e.retrieved = true rules := inspectionFilter{set: e.extractor.Rules} names := inspectionFilter{set: e.extractor.MetricNames} condition := e.timeRange.Condition() var finalRows [][]types.Datum for rule, tables := range inspectionSummaryRules { if len(rules.set) != 0 && !rules.set.Exist(rule) { continue } for _, name := range tables { if !names.enable(name) { continue } def, found := infoschema.MetricTableMap[name] if !found { sctx.GetSessionVars().StmtCtx.AppendWarning(fmt.Errorf("metrics table: %s not found", name)) continue } cols := def.Labels comment := def.Comment cond := condition if def.Quantile > 0 { cols = append(cols, "quantile") if len(e.extractor.Quantiles) > 0 { qs := make([]string, len(e.extractor.Quantiles)) for i, q := range e.extractor.Quantiles { qs[i] = fmt.Sprintf("%f", q) } cond += " and quantile in (" + strings.Join(qs, ",") + ")" } else { cond += " and quantile=0.99" } } var sql string if len(cols) > 0 { sql = fmt.Sprintf("select avg(value),min(value),max(value),`%s` from `%s`.`%s` %s group by `%[1]s` order by `%[1]s`", strings.Join(cols, "`,`"), util.MetricSchemaName.L, name, cond) } else { sql = fmt.Sprintf("select avg(value),min(value),max(value) from `%s`.`%s` %s", util.MetricSchemaName.L, name, cond) } rows, _, err := sctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQLWithContext(ctx, sql) if err != nil { return nil, errors.Errorf("execute '%s' failed: %v", sql, err) } nonInstanceLabelIndex := 0 if len(def.Labels) > 0 && def.Labels[0] == "instance" { nonInstanceLabelIndex = 1 } // skip min/max/avg const skipCols = 3 for _, row := range rows { instance := "" if nonInstanceLabelIndex > 0 { instance = row.GetString(skipCols) // skip min/max/avg } var labels []string for i, label := range def.Labels[nonInstanceLabelIndex:] { // skip min/max/avg/instance val := row.GetString(skipCols + nonInstanceLabelIndex + i) if label == "store" || label == "store_id" { val = fmt.Sprintf("store_id:%s", val) } labels = append(labels, val) } var quantile interface{} if def.Quantile > 0 { quantile = row.GetFloat64(row.Len() - 1) // quantile will be the last column } finalRows = append(finalRows, types.MakeDatums( rule, instance, name, strings.Join(labels, ", "), quantile, row.GetFloat64(0), // avg row.GetFloat64(1), // min row.GetFloat64(2), // max comment, )) } } } return finalRows, nil }