You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

241 lines
8.4 KiB

// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package collate
import (
"sort"
"sync/atomic"
"github.com/pingcap/errors"
"github.com/pingcap/parser/charset"
"github.com/pingcap/parser/mysql"
"github.com/pingcap/parser/terror"
"github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/logutil"
"go.uber.org/zap"
)
var (
newCollatorMap map[string]Collator
newCollatorIDMap map[int]Collator
newCollationEnabled int32
// binCollatorInstance is a singleton used for all collations when newCollationEnabled is false.
binCollatorInstance = &binCollator{}
// ErrUnsupportedCollation is returned when an unsupported collation is specified.
ErrUnsupportedCollation = dbterror.ClassDDL.NewStdErr(mysql.ErrUnknownCollation, mysql.Message("Unsupported collation when new collation is enabled: '%-.64s'", nil))
// ErrIllegalMixCollation is returned when illegal mix of collations.
ErrIllegalMixCollation = dbterror.ClassExpression.NewStd(mysql.ErrCantAggregate2collations)
)
// DefaultLen is set for datum if the string datum don't know its length.
const (
DefaultLen = 0
)
// Collator provides functionality for comparing strings for a given
// collation order.
type Collator interface {
// Compare returns an integer comparing the two strings. The result will be 0 if a == b, -1 if a < b, and +1 if a > b.
Compare(a, b string) int
// Key returns the collate key for str. If the collation is padding, make sure the PadLen >= len(rune[]str) in opt.
Key(str string) []byte
// Pattern get a collation-aware WildcardPattern.
Pattern() WildcardPattern
}
// WildcardPattern is the interface used for wildcard pattern match.
type WildcardPattern interface {
// Compile compiles the patternStr with specified escape character.
Compile(patternStr string, escape byte)
// DoMatch tries to match the str with compiled pattern, `Compile()` must be called before calling it.
DoMatch(str string) bool
}
// EnableNewCollations enables the new collation.
func EnableNewCollations() {
SetNewCollationEnabledForTest(true)
}
// SetNewCollationEnabledForTest sets if the new collation are enabled in test.
// Note: Be careful to use this function, if this functions is used in tests, make sure the tests are serial.
func SetNewCollationEnabledForTest(flag bool) {
if flag {
atomic.StoreInt32(&newCollationEnabled, 1)
return
}
atomic.StoreInt32(&newCollationEnabled, 0)
}
// NewCollationEnabled returns if the new collations are enabled.
func NewCollationEnabled() bool {
return atomic.LoadInt32(&newCollationEnabled) == 1
}
// CompatibleCollate checks whether the two collate are the same.
func CompatibleCollate(collate1, collate2 string) bool {
if (collate1 == "utf8mb4_general_ci" || collate1 == "utf8_general_ci") && (collate2 == "utf8mb4_general_ci" || collate2 == "utf8_general_ci") {
return true
} else if (collate1 == "utf8mb4_bin" || collate1 == "utf8_bin") && (collate2 == "utf8mb4_bin" || collate2 == "utf8_bin") {
return true
} else {
return collate1 == collate2
}
}
// RewriteNewCollationIDIfNeeded rewrites a collation id if the new collations are enabled.
// When new collations are enabled, we turn the collation id to negative so that other the
// components of the cluster(for example, TiKV) is able to aware of it without any change to
// the protocol definition.
// When new collations are not enabled, collation id remains the same.
func RewriteNewCollationIDIfNeeded(id int32) int32 {
if atomic.LoadInt32(&newCollationEnabled) == 1 {
if id < 0 {
logutil.BgLogger().Warn("Unexpected negative collation ID for rewrite.", zap.Int32("ID", id))
} else {
return -id
}
}
return id
}
// RestoreCollationIDIfNeeded restores a collation id if the new collations are enabled.
func RestoreCollationIDIfNeeded(id int32) int32 {
if atomic.LoadInt32(&newCollationEnabled) == 1 {
if id > 0 {
logutil.BgLogger().Warn("Unexpected positive collation ID for restore.", zap.Int32("ID", id))
} else {
return -id
}
}
return id
}
// GetCollator get the collator according to collate, it will return the binary collator if the corresponding collator doesn't exist.
func GetCollator(collate string) Collator {
if atomic.LoadInt32(&newCollationEnabled) == 1 {
ctor, ok := newCollatorMap[collate]
if !ok {
logutil.BgLogger().Warn(
"Unable to get collator by name, use binCollator instead.",
zap.String("name", collate),
zap.Stack("stack"))
return newCollatorMap["utf8mb4_bin"]
}
return ctor
}
return binCollatorInstance
}
// GetCollatorByID get the collator according to id, it will return the binary collator if the corresponding collator doesn't exist.
func GetCollatorByID(id int) Collator {
if atomic.LoadInt32(&newCollationEnabled) == 1 {
ctor, ok := newCollatorIDMap[id]
if !ok {
logutil.BgLogger().Warn(
"Unable to get collator by ID, use binCollator instead.",
zap.Int("ID", id),
zap.Stack("stack"))
return newCollatorMap["utf8mb4_bin"]
}
return ctor
}
return binCollatorInstance
}
// CollationID2Name return the collation name by the given id.
// If the id is not found in the map, the default collation is returned.
func CollationID2Name(id int32) string {
name, ok := mysql.Collations[uint8(id)]
if !ok {
// TODO(bb7133): fix repeating logs when the following code is uncommented.
//logutil.BgLogger().Warn(
// "Unable to get collation name from ID, use default collation instead.",
// zap.Int32("ID", id),
// zap.Stack("stack"))
return mysql.DefaultCollationName
}
return name
}
// GetCollationByName wraps charset.GetCollationByName, it checks the collation.
func GetCollationByName(name string) (coll *charset.Collation, err error) {
if coll, err = charset.GetCollationByName(name); err != nil {
return nil, errors.Trace(err)
}
if atomic.LoadInt32(&newCollationEnabled) == 1 {
if _, ok := newCollatorIDMap[coll.ID]; !ok {
return nil, ErrUnsupportedCollation.GenWithStackByArgs(name)
}
}
return
}
// GetSupportedCollations gets information for all collations supported so far.
func GetSupportedCollations() []*charset.Collation {
if atomic.LoadInt32(&newCollationEnabled) == 1 {
newSupportedCollations := make([]*charset.Collation, 0, len(newCollatorMap))
for name := range newCollatorMap {
if coll, err := charset.GetCollationByName(name); err != nil {
// Should never happens.
terror.Log(err)
} else {
newSupportedCollations = append(newSupportedCollations, coll)
}
}
sort.Slice(newSupportedCollations, func(i int, j int) bool {
return newSupportedCollations[i].Name < newSupportedCollations[j].Name
})
return newSupportedCollations
}
return charset.GetSupportedCollations()
}
func truncateTailingSpace(str string) string {
byteLen := len(str)
i := byteLen - 1
for ; i >= 0; i-- {
if str[i] != ' ' {
break
}
}
str = str[:i+1]
return str
}
// IsCICollation returns if the collation is case-sensitive
func IsCICollation(collate string) bool {
return collate == "utf8_general_ci" || collate == "utf8mb4_general_ci"
}
func init() {
newCollatorMap = make(map[string]Collator)
newCollatorIDMap = make(map[int]Collator)
newCollatorMap["binary"] = &binCollator{}
newCollatorIDMap[int(mysql.CollationNames["binary"])] = &binCollator{}
newCollatorMap["ascii_bin"] = &binPaddingCollator{}
newCollatorIDMap[int(mysql.CollationNames["ascii_bin"])] = &binPaddingCollator{}
newCollatorMap["latin1_bin"] = &binPaddingCollator{}
newCollatorIDMap[int(mysql.CollationNames["latin1_bin"])] = &binPaddingCollator{}
newCollatorMap["utf8mb4_bin"] = &binPaddingCollator{}
newCollatorIDMap[int(mysql.CollationNames["utf8mb4_bin"])] = &binPaddingCollator{}
newCollatorMap["utf8_bin"] = &binPaddingCollator{}
newCollatorIDMap[int(mysql.CollationNames["utf8_bin"])] = &binPaddingCollator{}
newCollatorMap["utf8mb4_general_ci"] = &generalCICollator{}
newCollatorIDMap[int(mysql.CollationNames["utf8mb4_general_ci"])] = &generalCICollator{}
newCollatorMap["utf8_general_ci"] = &generalCICollator{}
newCollatorIDMap[int(mysql.CollationNames["utf8_general_ci"])] = &generalCICollator{}
}