You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
288 lines
10 KiB
288 lines
10 KiB
// Copyright 2018 PingCAP, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package statistics
|
|
|
|
import (
|
|
"bytes"
|
|
|
|
. "github.com/pingcap/check"
|
|
"github.com/pingcap/parser/mysql"
|
|
"github.com/pingcap/tidb/types"
|
|
"github.com/pingcap/tidb/util/codec"
|
|
)
|
|
|
|
var _ = Suite(&testFeedbackSuite{})
|
|
|
|
type testFeedbackSuite struct {
|
|
}
|
|
|
|
func newFeedback(lower, upper, count int64) Feedback {
|
|
low, upp := types.NewIntDatum(lower), types.NewIntDatum(upper)
|
|
return Feedback{&low, &upp, count, 0}
|
|
}
|
|
|
|
func genFeedbacks(lower, upper int64) []Feedback {
|
|
var feedbacks []Feedback
|
|
for i := lower; i < upper; i++ {
|
|
feedbacks = append(feedbacks, newFeedback(i, upper, upper-i+1))
|
|
}
|
|
return feedbacks
|
|
}
|
|
|
|
func appendBucket(h *Histogram, l, r int64) {
|
|
lower, upper := types.NewIntDatum(l), types.NewIntDatum(r)
|
|
h.AppendBucket(&lower, &upper, 0, 0)
|
|
}
|
|
|
|
func genHistogram() *Histogram {
|
|
h := NewHistogram(0, 0, 0, 0, types.NewFieldType(mysql.TypeLong), 5, 0)
|
|
appendBucket(h, 1, 1)
|
|
appendBucket(h, 2, 3)
|
|
appendBucket(h, 5, 7)
|
|
appendBucket(h, 10, 20)
|
|
appendBucket(h, 30, 50)
|
|
return h
|
|
}
|
|
|
|
func (s *testFeedbackSuite) TestUpdateHistogram(c *C) {
|
|
feedbacks := []Feedback{
|
|
newFeedback(0, 1, 10000),
|
|
newFeedback(1, 2, 1),
|
|
newFeedback(2, 3, 3),
|
|
newFeedback(4, 5, 2),
|
|
newFeedback(5, 7, 4),
|
|
}
|
|
feedbacks = append(feedbacks, genFeedbacks(8, 20)...)
|
|
feedbacks = append(feedbacks, genFeedbacks(21, 60)...)
|
|
|
|
q := NewQueryFeedback(0, genHistogram(), 0, false)
|
|
q.Feedback = feedbacks
|
|
originBucketCount := defaultBucketCount
|
|
defaultBucketCount = 7
|
|
defer func() { defaultBucketCount = originBucketCount }()
|
|
c.Assert(UpdateHistogram(q.Hist, q).ToString(0), Equals,
|
|
"column:0 ndv:10053 totColSize:0\n"+
|
|
"num: 10001 lower_bound: 0 upper_bound: 2 repeats: 0\n"+
|
|
"num: 7 lower_bound: 2 upper_bound: 5 repeats: 0\n"+
|
|
"num: 4 lower_bound: 5 upper_bound: 7 repeats: 0\n"+
|
|
"num: 11 lower_bound: 10 upper_bound: 20 repeats: 0\n"+
|
|
"num: 19 lower_bound: 30 upper_bound: 49 repeats: 0\n"+
|
|
"num: 11 lower_bound: 50 upper_bound: 60 repeats: 0")
|
|
}
|
|
|
|
func (s *testFeedbackSuite) TestSplitBuckets(c *C) {
|
|
// test bucket split
|
|
feedbacks := []Feedback{newFeedback(0, 1, 1)}
|
|
for i := 0; i < 100; i++ {
|
|
feedbacks = append(feedbacks, newFeedback(10, 15, 5))
|
|
}
|
|
q := NewQueryFeedback(0, genHistogram(), 0, false)
|
|
q.Feedback = feedbacks
|
|
buckets, isNewBuckets, totalCount := splitBuckets(q.Hist, q)
|
|
c.Assert(buildNewHistogram(q.Hist, buckets).ToString(0), Equals,
|
|
"column:0 ndv:0 totColSize:0\n"+
|
|
"num: 1 lower_bound: 0 upper_bound: 1 repeats: 0\n"+
|
|
"num: 0 lower_bound: 2 upper_bound: 3 repeats: 0\n"+
|
|
"num: 0 lower_bound: 5 upper_bound: 7 repeats: 0\n"+
|
|
"num: 5 lower_bound: 10 upper_bound: 15 repeats: 0\n"+
|
|
"num: 0 lower_bound: 16 upper_bound: 20 repeats: 0\n"+
|
|
"num: 0 lower_bound: 30 upper_bound: 50 repeats: 0")
|
|
c.Assert(isNewBuckets, DeepEquals, []bool{false, false, false, true, true, false})
|
|
c.Assert(totalCount, Equals, int64(6))
|
|
|
|
// test do not split if the bucket count is too small
|
|
feedbacks = []Feedback{newFeedback(0, 1, 100000)}
|
|
for i := 0; i < 100; i++ {
|
|
feedbacks = append(feedbacks, newFeedback(10, 15, 1))
|
|
}
|
|
q = NewQueryFeedback(0, genHistogram(), 0, false)
|
|
q.Feedback = feedbacks
|
|
buckets, isNewBuckets, totalCount = splitBuckets(q.Hist, q)
|
|
c.Assert(buildNewHistogram(q.Hist, buckets).ToString(0), Equals,
|
|
"column:0 ndv:0 totColSize:0\n"+
|
|
"num: 100000 lower_bound: 0 upper_bound: 1 repeats: 0\n"+
|
|
"num: 0 lower_bound: 2 upper_bound: 3 repeats: 0\n"+
|
|
"num: 0 lower_bound: 5 upper_bound: 7 repeats: 0\n"+
|
|
"num: 1 lower_bound: 10 upper_bound: 15 repeats: 0\n"+
|
|
"num: 0 lower_bound: 16 upper_bound: 20 repeats: 0\n"+
|
|
"num: 0 lower_bound: 30 upper_bound: 50 repeats: 0")
|
|
c.Assert(isNewBuckets, DeepEquals, []bool{false, false, false, true, true, false})
|
|
c.Assert(totalCount, Equals, int64(100001))
|
|
|
|
// test do not split if the result bucket count is too small
|
|
h := NewHistogram(0, 0, 0, 0, types.NewFieldType(mysql.TypeLong), 5, 0)
|
|
appendBucket(h, 0, 1000000)
|
|
h.Buckets[0].Count = 1000000
|
|
feedbacks = feedbacks[:0]
|
|
for i := 0; i < 100; i++ {
|
|
feedbacks = append(feedbacks, newFeedback(0, 10, 1))
|
|
}
|
|
q = NewQueryFeedback(0, h, 0, false)
|
|
q.Feedback = feedbacks
|
|
buckets, isNewBuckets, totalCount = splitBuckets(q.Hist, q)
|
|
c.Assert(buildNewHistogram(q.Hist, buckets).ToString(0), Equals,
|
|
"column:0 ndv:0 totColSize:0\n"+
|
|
"num: 1000000 lower_bound: 0 upper_bound: 1000000 repeats: 0")
|
|
c.Assert(isNewBuckets, DeepEquals, []bool{false})
|
|
c.Assert(totalCount, Equals, int64(1000000))
|
|
|
|
// test split even if the feedback range is too small
|
|
h = NewHistogram(0, 0, 0, 0, types.NewFieldType(mysql.TypeLong), 5, 0)
|
|
appendBucket(h, 0, 1000000)
|
|
feedbacks = feedbacks[:0]
|
|
for i := 0; i < 100; i++ {
|
|
feedbacks = append(feedbacks, newFeedback(0, 10, 1))
|
|
}
|
|
q = NewQueryFeedback(0, h, 0, false)
|
|
q.Feedback = feedbacks
|
|
buckets, isNewBuckets, totalCount = splitBuckets(q.Hist, q)
|
|
c.Assert(buildNewHistogram(q.Hist, buckets).ToString(0), Equals,
|
|
"column:0 ndv:0 totColSize:0\n"+
|
|
"num: 1 lower_bound: 0 upper_bound: 10 repeats: 0\n"+
|
|
"num: 0 lower_bound: 11 upper_bound: 1000000 repeats: 0")
|
|
c.Assert(isNewBuckets, DeepEquals, []bool{true, true})
|
|
c.Assert(totalCount, Equals, int64(1))
|
|
|
|
// test merge the non-overlapped feedbacks.
|
|
h = NewHistogram(0, 0, 0, 0, types.NewFieldType(mysql.TypeLong), 5, 0)
|
|
appendBucket(h, 0, 10000)
|
|
feedbacks = feedbacks[:0]
|
|
feedbacks = append(feedbacks, newFeedback(0, 4000, 4000))
|
|
feedbacks = append(feedbacks, newFeedback(4001, 9999, 1000))
|
|
q = NewQueryFeedback(0, h, 0, false)
|
|
q.Feedback = feedbacks
|
|
buckets, isNewBuckets, totalCount = splitBuckets(q.Hist, q)
|
|
c.Assert(buildNewHistogram(q.Hist, buckets).ToString(0), Equals,
|
|
"column:0 ndv:0 totColSize:0\n"+
|
|
"num: 5001 lower_bound: 0 upper_bound: 10000 repeats: 0")
|
|
c.Assert(isNewBuckets, DeepEquals, []bool{false})
|
|
c.Assert(totalCount, Equals, int64(5001))
|
|
}
|
|
|
|
func (s *testFeedbackSuite) TestMergeBuckets(c *C) {
|
|
originBucketCount := defaultBucketCount
|
|
defer func() { defaultBucketCount = originBucketCount }()
|
|
tests := []struct {
|
|
points []int64
|
|
counts []int64
|
|
isNewBuckets []bool
|
|
bucketCount int
|
|
result string
|
|
}{
|
|
{
|
|
points: []int64{1, 2},
|
|
counts: []int64{1},
|
|
isNewBuckets: []bool{false},
|
|
bucketCount: 1,
|
|
result: "column:0 ndv:0 totColSize:0\nnum: 1 lower_bound: 1 upper_bound: 2 repeats: 0",
|
|
},
|
|
{
|
|
points: []int64{1, 2, 2, 3, 3, 4},
|
|
counts: []int64{100000, 1, 1},
|
|
isNewBuckets: []bool{false, false, false},
|
|
bucketCount: 2,
|
|
result: "column:0 ndv:0 totColSize:0\n" +
|
|
"num: 100000 lower_bound: 1 upper_bound: 2 repeats: 0\n" +
|
|
"num: 2 lower_bound: 2 upper_bound: 4 repeats: 0",
|
|
},
|
|
// test do not Merge if the result bucket count is too large
|
|
{
|
|
points: []int64{1, 2, 2, 3, 3, 4, 4, 5},
|
|
counts: []int64{1, 1, 100000, 100000},
|
|
isNewBuckets: []bool{false, false, false, false},
|
|
bucketCount: 3,
|
|
result: "column:0 ndv:0 totColSize:0\n" +
|
|
"num: 2 lower_bound: 1 upper_bound: 3 repeats: 0\n" +
|
|
"num: 100000 lower_bound: 3 upper_bound: 4 repeats: 0\n" +
|
|
"num: 100000 lower_bound: 4 upper_bound: 5 repeats: 0",
|
|
},
|
|
}
|
|
for _, t := range tests {
|
|
bkts := make([]bucket, 0, len(t.counts))
|
|
totalCount := int64(0)
|
|
for i := 0; i < len(t.counts); i++ {
|
|
lower, upper := types.NewIntDatum(t.points[2*i]), types.NewIntDatum(t.points[2*i+1])
|
|
bkts = append(bkts, bucket{&lower, &upper, t.counts[i], 0})
|
|
totalCount += t.counts[i]
|
|
}
|
|
defaultBucketCount = t.bucketCount
|
|
bkts = mergeBuckets(bkts, t.isNewBuckets, float64(totalCount))
|
|
result := buildNewHistogram(&Histogram{Tp: types.NewFieldType(mysql.TypeLong)}, bkts).ToString(0)
|
|
c.Assert(result, Equals, t.result)
|
|
}
|
|
}
|
|
|
|
func encodeInt(v int64) *types.Datum {
|
|
val := codec.EncodeInt(nil, v)
|
|
d := types.NewBytesDatum(val)
|
|
return &d
|
|
}
|
|
|
|
func (s *testFeedbackSuite) TestFeedbackEncoding(c *C) {
|
|
hist := NewHistogram(0, 0, 0, 0, types.NewFieldType(mysql.TypeLong), 0, 0)
|
|
q := &QueryFeedback{Hist: hist, Tp: PkType}
|
|
q.Feedback = append(q.Feedback, Feedback{encodeInt(0), encodeInt(3), 1, 0})
|
|
q.Feedback = append(q.Feedback, Feedback{encodeInt(0), encodeInt(5), 1, 0})
|
|
val, err := EncodeFeedback(q)
|
|
c.Assert(err, IsNil)
|
|
rq := &QueryFeedback{}
|
|
c.Assert(DecodeFeedback(val, rq, nil, hist.Tp), IsNil)
|
|
for _, fb := range rq.Feedback {
|
|
fb.Lower.SetBytes(codec.EncodeInt(nil, fb.Lower.GetInt64()))
|
|
fb.Upper.SetBytes(codec.EncodeInt(nil, fb.Upper.GetInt64()))
|
|
}
|
|
c.Assert(q.Equal(rq), IsTrue)
|
|
|
|
hist.Tp = types.NewFieldType(mysql.TypeBlob)
|
|
q = &QueryFeedback{Hist: hist}
|
|
q.Feedback = append(q.Feedback, Feedback{encodeInt(0), encodeInt(3), 1, 0})
|
|
q.Feedback = append(q.Feedback, Feedback{encodeInt(0), encodeInt(1), 1, 0})
|
|
val, err = EncodeFeedback(q)
|
|
c.Assert(err, IsNil)
|
|
rq = &QueryFeedback{}
|
|
cms := NewCMSketch(4, 4)
|
|
c.Assert(DecodeFeedback(val, rq, cms, hist.Tp), IsNil)
|
|
c.Assert(cms.QueryBytes(codec.EncodeInt(nil, 0)), Equals, uint64(1))
|
|
q.Feedback = q.Feedback[:1]
|
|
c.Assert(q.Equal(rq), IsTrue)
|
|
}
|
|
|
|
// Equal tests if two query feedback equal, it is only used in test.
|
|
func (q *QueryFeedback) Equal(rq *QueryFeedback) bool {
|
|
if len(q.Feedback) != len(rq.Feedback) {
|
|
return false
|
|
}
|
|
for i, fb := range q.Feedback {
|
|
rfb := rq.Feedback[i]
|
|
if fb.Count != rfb.Count {
|
|
return false
|
|
}
|
|
if fb.Lower.Kind() == types.KindInt64 {
|
|
if fb.Lower.GetInt64() != rfb.Lower.GetInt64() {
|
|
return false
|
|
}
|
|
if fb.Upper.GetInt64() != rfb.Upper.GetInt64() {
|
|
return false
|
|
}
|
|
} else {
|
|
if !bytes.Equal(fb.Lower.GetBytes(), rfb.Lower.GetBytes()) {
|
|
return false
|
|
}
|
|
if !bytes.Equal(fb.Upper.GetBytes(), rfb.Upper.GetBytes()) {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|