You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
410 lines
10 KiB
410 lines
10 KiB
// Copyright 2017 PingCAP, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package filesort
|
|
|
|
import (
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
. "github.com/pingcap/check"
|
|
"github.com/pingcap/tidb/sessionctx/stmtctx"
|
|
"github.com/pingcap/tidb/types"
|
|
"github.com/pingcap/tidb/util/testleak"
|
|
)
|
|
|
|
func TestT(t *testing.T) {
|
|
CustomVerboseFlag = true
|
|
TestingT(t)
|
|
}
|
|
|
|
var _ = Suite(&testFileSortSuite{})
|
|
|
|
type testFileSortSuite struct {
|
|
}
|
|
|
|
func nextRow(r *rand.Rand, keySize int, valSize int) (key []types.Datum, val []types.Datum, handle int64) {
|
|
key = make([]types.Datum, keySize)
|
|
for i := range key {
|
|
key[i] = types.NewDatum(r.Int())
|
|
}
|
|
|
|
val = make([]types.Datum, valSize)
|
|
for j := range val {
|
|
val[j] = types.NewDatum(r.Int())
|
|
}
|
|
|
|
handle = r.Int63()
|
|
return
|
|
}
|
|
|
|
func (s *testFileSortSuite) TestLessThan(c *C) {
|
|
defer testleak.AfterTest(c)()
|
|
|
|
sc := new(stmtctx.StatementContext)
|
|
|
|
d0 := types.NewDatum(0)
|
|
d1 := types.NewDatum(1)
|
|
|
|
tblOneColumn := []struct {
|
|
Arg1 []types.Datum
|
|
Arg2 []types.Datum
|
|
Arg3 []bool
|
|
Ret bool
|
|
}{
|
|
{[]types.Datum{d0}, []types.Datum{d0}, []bool{false}, false},
|
|
{[]types.Datum{d0}, []types.Datum{d1}, []bool{false}, true},
|
|
{[]types.Datum{d1}, []types.Datum{d0}, []bool{false}, false},
|
|
{[]types.Datum{d0}, []types.Datum{d0}, []bool{true}, false},
|
|
{[]types.Datum{d0}, []types.Datum{d1}, []bool{true}, false},
|
|
{[]types.Datum{d1}, []types.Datum{d0}, []bool{true}, true},
|
|
}
|
|
|
|
for _, t := range tblOneColumn {
|
|
ret, err := lessThan(sc, t.Arg1, t.Arg2, t.Arg3)
|
|
c.Assert(err, IsNil)
|
|
c.Assert(ret, Equals, t.Ret)
|
|
}
|
|
|
|
tblTwoColumns := []struct {
|
|
Arg1 []types.Datum
|
|
Arg2 []types.Datum
|
|
Arg3 []bool
|
|
Ret bool
|
|
}{
|
|
{[]types.Datum{d0, d0}, []types.Datum{d1, d1}, []bool{false, false}, true},
|
|
{[]types.Datum{d0, d1}, []types.Datum{d1, d1}, []bool{false, false}, true},
|
|
{[]types.Datum{d0, d0}, []types.Datum{d1, d1}, []bool{false, false}, true},
|
|
{[]types.Datum{d0, d0}, []types.Datum{d0, d1}, []bool{false, false}, true},
|
|
{[]types.Datum{d0, d1}, []types.Datum{d0, d1}, []bool{false, false}, false},
|
|
{[]types.Datum{d0, d1}, []types.Datum{d0, d0}, []bool{false, false}, false},
|
|
{[]types.Datum{d1, d0}, []types.Datum{d0, d1}, []bool{false, false}, false},
|
|
{[]types.Datum{d1, d1}, []types.Datum{d0, d1}, []bool{false, false}, false},
|
|
{[]types.Datum{d1, d1}, []types.Datum{d0, d0}, []bool{false, false}, false},
|
|
}
|
|
|
|
for _, t := range tblTwoColumns {
|
|
ret, err := lessThan(sc, t.Arg1, t.Arg2, t.Arg3)
|
|
c.Assert(err, IsNil)
|
|
c.Assert(ret, Equals, t.Ret)
|
|
}
|
|
}
|
|
|
|
func (s *testFileSortSuite) TestInMemory(c *C) {
|
|
defer testleak.AfterTest(c)()
|
|
|
|
seed := rand.NewSource(time.Now().UnixNano())
|
|
r := rand.New(seed)
|
|
|
|
sc := new(stmtctx.StatementContext)
|
|
keySize := r.Intn(10) + 1 // random int in range [1, 10]
|
|
valSize := r.Intn(20) + 1 // random int in range [1, 20]
|
|
bufSize := 40 // hold up to 40 items per file
|
|
byDesc := make([]bool, keySize)
|
|
for i := range byDesc {
|
|
byDesc[i] = r.Intn(2) == 0
|
|
}
|
|
|
|
var (
|
|
err error
|
|
fs *FileSorter
|
|
pkey []types.Datum
|
|
key []types.Datum
|
|
tmpDir string
|
|
ret bool
|
|
)
|
|
|
|
tmpDir, err = ioutil.TempDir("", "util_filesort_test")
|
|
c.Assert(err, IsNil)
|
|
|
|
fsBuilder := new(Builder)
|
|
fs, err = fsBuilder.SetSC(sc).SetSchema(keySize, valSize).SetBuf(bufSize).SetWorkers(1).SetDesc(byDesc).SetDir(tmpDir).Build()
|
|
c.Assert(err, IsNil)
|
|
defer fs.Close()
|
|
|
|
nRows := r.Intn(bufSize-1) + 1 // random int in range [1, bufSize - 1]
|
|
for i := 1; i <= nRows; i++ {
|
|
err = fs.Input(nextRow(r, keySize, valSize))
|
|
c.Assert(err, IsNil)
|
|
}
|
|
|
|
pkey, _, _, err = fs.Output()
|
|
c.Assert(err, IsNil)
|
|
for i := 1; i < nRows; i++ {
|
|
key, _, _, err = fs.Output()
|
|
c.Assert(err, IsNil)
|
|
ret, err = lessThan(sc, key, pkey, byDesc)
|
|
c.Assert(err, IsNil)
|
|
c.Assert(ret, IsFalse)
|
|
pkey = key
|
|
}
|
|
}
|
|
|
|
func (s *testFileSortSuite) TestMultipleFiles(c *C) {
|
|
defer testleak.AfterTest(c)()
|
|
|
|
seed := rand.NewSource(time.Now().UnixNano())
|
|
r := rand.New(seed)
|
|
|
|
sc := new(stmtctx.StatementContext)
|
|
keySize := r.Intn(10) + 1 // random int in range [1, 10]
|
|
valSize := r.Intn(20) + 1 // random int in range [1, 20]
|
|
bufSize := 40 // hold up to 40 items per file
|
|
byDesc := make([]bool, keySize)
|
|
for i := range byDesc {
|
|
byDesc[i] = r.Intn(2) == 0
|
|
}
|
|
|
|
var (
|
|
err error
|
|
fs *FileSorter
|
|
pkey []types.Datum
|
|
key []types.Datum
|
|
tmpDir string
|
|
ret bool
|
|
)
|
|
|
|
tmpDir, err = ioutil.TempDir("", "util_filesort_test")
|
|
c.Assert(err, IsNil)
|
|
|
|
fsBuilder := new(Builder)
|
|
|
|
// Test for basic function.
|
|
_, err = fsBuilder.Build()
|
|
c.Assert(err.Error(), Equals, "StatementContext is nil")
|
|
fsBuilder.SetSC(sc)
|
|
_, err = fsBuilder.Build()
|
|
c.Assert(err.Error(), Equals, "key size is not positive")
|
|
fsBuilder.SetDesc(byDesc)
|
|
_, err = fsBuilder.Build()
|
|
c.Assert(err.Error(), Equals, "mismatch in key size and byDesc slice")
|
|
fsBuilder.SetSchema(keySize, valSize)
|
|
_, err = fsBuilder.Build()
|
|
c.Assert(err.Error(), Equals, "buffer size is not positive")
|
|
fsBuilder.SetBuf(bufSize)
|
|
_, err = fsBuilder.Build()
|
|
c.Assert(err.Error(), Equals, "tmpDir does not exist")
|
|
fsBuilder.SetDir(tmpDir)
|
|
|
|
fs, err = fsBuilder.SetWorkers(1).Build()
|
|
c.Assert(err, IsNil)
|
|
defer fs.Close()
|
|
|
|
nRows := (r.Intn(bufSize) + 1) * (r.Intn(10) + 2)
|
|
for i := 1; i <= nRows; i++ {
|
|
err = fs.Input(nextRow(r, keySize, valSize))
|
|
c.Assert(err, IsNil)
|
|
}
|
|
|
|
pkey, _, _, err = fs.Output()
|
|
c.Assert(err, IsNil)
|
|
for i := 1; i < nRows; i++ {
|
|
key, _, _, err = fs.Output()
|
|
c.Assert(err, IsNil)
|
|
ret, err = lessThan(sc, key, pkey, byDesc)
|
|
c.Assert(err, IsNil)
|
|
c.Assert(ret, IsFalse)
|
|
pkey = key
|
|
}
|
|
}
|
|
|
|
func (s *testFileSortSuite) TestMultipleWorkers(c *C) {
|
|
defer testleak.AfterTest(c)()
|
|
|
|
seed := rand.NewSource(time.Now().UnixNano())
|
|
r := rand.New(seed)
|
|
|
|
sc := new(stmtctx.StatementContext)
|
|
keySize := r.Intn(10) + 1 // random int in range [1, 10]
|
|
valSize := r.Intn(20) + 1 // random int in range [1, 20]
|
|
bufSize := 40 // hold up to 40 items per file
|
|
byDesc := make([]bool, keySize)
|
|
for i := range byDesc {
|
|
byDesc[i] = r.Intn(2) == 0
|
|
}
|
|
|
|
var (
|
|
err error
|
|
fs *FileSorter
|
|
pkey []types.Datum
|
|
key []types.Datum
|
|
tmpDir string
|
|
ret bool
|
|
)
|
|
|
|
tmpDir, err = ioutil.TempDir("", "util_filesort_test")
|
|
c.Assert(err, IsNil)
|
|
|
|
fsBuilder := new(Builder)
|
|
fs, err = fsBuilder.SetSC(sc).SetSchema(keySize, valSize).SetBuf(bufSize).SetWorkers(4).SetDesc(byDesc).SetDir(tmpDir).Build()
|
|
c.Assert(err, IsNil)
|
|
defer fs.Close()
|
|
|
|
nRows := (r.Intn(bufSize) + 1) * (r.Intn(10) + 2)
|
|
for i := 1; i <= nRows; i++ {
|
|
err = fs.Input(nextRow(r, keySize, valSize))
|
|
c.Assert(err, IsNil)
|
|
}
|
|
|
|
pkey, _, _, err = fs.Output()
|
|
c.Assert(err, IsNil)
|
|
for i := 1; i < nRows; i++ {
|
|
key, _, _, err = fs.Output()
|
|
c.Assert(err, IsNil)
|
|
ret, err = lessThan(sc, key, pkey, byDesc)
|
|
c.Assert(err, IsNil)
|
|
c.Assert(ret, IsFalse)
|
|
pkey = key
|
|
}
|
|
}
|
|
|
|
func (s *testFileSortSuite) TestClose(c *C) {
|
|
defer testleak.AfterTest(c)()
|
|
|
|
seed := rand.NewSource(time.Now().UnixNano())
|
|
r := rand.New(seed)
|
|
|
|
sc := new(stmtctx.StatementContext)
|
|
keySize := 2
|
|
valSize := 2
|
|
bufSize := 40
|
|
byDesc := []bool{false, false}
|
|
|
|
var (
|
|
err error
|
|
fs0 *FileSorter
|
|
fs1 *FileSorter
|
|
tmpDir0 string
|
|
tmpDir1 string
|
|
errmsg = "FileSorter has been closed"
|
|
)
|
|
|
|
// Prepare two FileSorter instances for tests
|
|
fsBuilder := new(Builder)
|
|
tmpDir0, err = ioutil.TempDir("", "util_filesort_test")
|
|
c.Assert(err, IsNil)
|
|
fs0, err = fsBuilder.SetSC(sc).SetSchema(keySize, valSize).SetBuf(bufSize).SetWorkers(1).SetDesc(byDesc).SetDir(tmpDir0).Build()
|
|
c.Assert(err, IsNil)
|
|
defer fs0.Close()
|
|
|
|
tmpDir1, err = ioutil.TempDir("", "util_filesort_test")
|
|
c.Assert(err, IsNil)
|
|
fs1, err = fsBuilder.SetSC(sc).SetSchema(keySize, valSize).SetBuf(bufSize).SetWorkers(1).SetDesc(byDesc).SetDir(tmpDir1).Build()
|
|
c.Assert(err, IsNil)
|
|
defer fs1.Close()
|
|
|
|
// 1. Close after some Input
|
|
err = fs0.Input(nextRow(r, keySize, valSize))
|
|
c.Assert(err, IsNil)
|
|
|
|
err = fs0.Close()
|
|
c.Assert(err, IsNil)
|
|
|
|
_, err = os.Stat(tmpDir0)
|
|
c.Assert(os.IsNotExist(err), IsTrue)
|
|
|
|
_, _, _, err = fs0.Output()
|
|
c.Assert(err, ErrorMatches, errmsg)
|
|
|
|
err = fs0.Input(nextRow(r, keySize, valSize))
|
|
c.Assert(err, ErrorMatches, errmsg)
|
|
|
|
err = fs0.Close()
|
|
c.Assert(err, IsNil)
|
|
|
|
// 2. Close after some Output
|
|
err = fs1.Input(nextRow(r, keySize, valSize))
|
|
c.Assert(err, IsNil)
|
|
err = fs1.Input(nextRow(r, keySize, valSize))
|
|
c.Assert(err, IsNil)
|
|
|
|
_, _, _, err = fs1.Output()
|
|
c.Assert(err, IsNil)
|
|
|
|
err = fs1.Close()
|
|
c.Assert(err, IsNil)
|
|
|
|
_, err = os.Stat(tmpDir1)
|
|
c.Assert(os.IsNotExist(err), IsTrue)
|
|
|
|
_, _, _, err = fs1.Output()
|
|
c.Assert(err, ErrorMatches, errmsg)
|
|
|
|
err = fs1.Input(nextRow(r, keySize, valSize))
|
|
c.Assert(err, ErrorMatches, errmsg)
|
|
|
|
err = fs1.Close()
|
|
c.Assert(err, IsNil)
|
|
}
|
|
|
|
func (s *testFileSortSuite) TestMismatchedUsage(c *C) {
|
|
defer testleak.AfterTest(c)()
|
|
|
|
seed := rand.NewSource(time.Now().UnixNano())
|
|
r := rand.New(seed)
|
|
|
|
sc := new(stmtctx.StatementContext)
|
|
keySize := 2
|
|
valSize := 2
|
|
bufSize := 40
|
|
byDesc := []bool{false, false}
|
|
|
|
var (
|
|
err error
|
|
fs0 *FileSorter
|
|
fs1 *FileSorter
|
|
key []types.Datum
|
|
tmpDir string
|
|
errmsg = "call input after output"
|
|
)
|
|
|
|
// Prepare two FileSorter instances for tests
|
|
fsBuilder := new(Builder)
|
|
tmpDir, err = ioutil.TempDir("", "util_filesort_test")
|
|
c.Assert(err, IsNil)
|
|
fs0, err = fsBuilder.SetSC(sc).SetSchema(keySize, valSize).SetBuf(bufSize).SetWorkers(1).SetDesc(byDesc).SetDir(tmpDir).Build()
|
|
c.Assert(err, IsNil)
|
|
defer fs0.Close()
|
|
|
|
tmpDir, err = ioutil.TempDir("", "util_filesort_test")
|
|
c.Assert(err, IsNil)
|
|
fs1, err = fsBuilder.SetSC(sc).SetSchema(keySize, valSize).SetBuf(bufSize).SetWorkers(1).SetDesc(byDesc).SetDir(tmpDir).Build()
|
|
c.Assert(err, IsNil)
|
|
defer fs1.Close()
|
|
|
|
// 1. call Output after fetched all rows
|
|
err = fs0.Input(nextRow(r, keySize, valSize))
|
|
c.Assert(err, IsNil)
|
|
|
|
key, _, _, err = fs0.Output()
|
|
c.Assert(err, IsNil)
|
|
c.Assert(key, NotNil)
|
|
|
|
key, _, _, err = fs0.Output()
|
|
c.Assert(err, IsNil)
|
|
c.Assert(key, IsNil)
|
|
|
|
// 2. call Input after Output
|
|
err = fs1.Input(nextRow(r, keySize, valSize))
|
|
c.Assert(err, IsNil)
|
|
|
|
key, _, _, err = fs1.Output()
|
|
c.Assert(err, IsNil)
|
|
c.Assert(key, NotNil)
|
|
|
|
err = fs1.Input(nextRow(r, keySize, valSize))
|
|
c.Assert(err, ErrorMatches, errmsg)
|
|
}
|
|
|