You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1006 lines
30 KiB
1006 lines
30 KiB
// Copyright 2016 PingCAP, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package tablecodec
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/binary"
|
|
"math"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"github.com/pingcap/errors"
|
|
"github.com/pingcap/parser/charset"
|
|
"github.com/pingcap/parser/model"
|
|
"github.com/pingcap/parser/mysql"
|
|
"github.com/pingcap/parser/terror"
|
|
"github.com/pingcap/tidb/errno"
|
|
"github.com/pingcap/tidb/kv"
|
|
"github.com/pingcap/tidb/sessionctx/stmtctx"
|
|
"github.com/pingcap/tidb/structure"
|
|
"github.com/pingcap/tidb/types"
|
|
"github.com/pingcap/tidb/util/codec"
|
|
"github.com/pingcap/tidb/util/collate"
|
|
"github.com/pingcap/tidb/util/dbterror"
|
|
"github.com/pingcap/tidb/util/rowcodec"
|
|
)
|
|
|
|
var (
|
|
errInvalidKey = dbterror.ClassXEval.NewStd(errno.ErrInvalidKey)
|
|
errInvalidRecordKey = dbterror.ClassXEval.NewStd(errno.ErrInvalidRecordKey)
|
|
errInvalidIndexKey = dbterror.ClassXEval.NewStd(errno.ErrInvalidIndexKey)
|
|
)
|
|
|
|
var (
|
|
tablePrefix = []byte{'t'}
|
|
recordPrefixSep = []byte("_r")
|
|
indexPrefixSep = []byte("_i")
|
|
metaPrefix = []byte{'m'}
|
|
)
|
|
|
|
const (
|
|
idLen = 8
|
|
prefixLen = 1 + idLen /*tableID*/ + 2
|
|
// RecordRowKeyLen is public for calculating avgerage row size.
|
|
RecordRowKeyLen = prefixLen + idLen /*handle*/
|
|
tablePrefixLength = 1
|
|
recordPrefixSepLength = 2
|
|
metaPrefixLength = 1
|
|
// MaxOldEncodeValueLen is the maximum len of the old encoding of index value.
|
|
MaxOldEncodeValueLen = 9
|
|
)
|
|
|
|
// TableSplitKeyLen is the length of key 't{table_id}' which is used for table split.
|
|
const TableSplitKeyLen = 1 + idLen
|
|
|
|
// TablePrefix returns table's prefix 't'.
|
|
func TablePrefix() []byte {
|
|
return tablePrefix
|
|
}
|
|
|
|
// EncodeRowKey encodes the table id and record handle into a kv.Key
|
|
func EncodeRowKey(tableID int64, encodedHandle []byte) kv.Key {
|
|
buf := make([]byte, 0, RecordRowKeyLen)
|
|
buf = appendTableRecordPrefix(buf, tableID)
|
|
buf = append(buf, encodedHandle...)
|
|
return buf
|
|
}
|
|
|
|
// EncodeRowKeyWithHandle encodes the table id, row handle into a kv.Key
|
|
func EncodeRowKeyWithHandle(tableID int64, handle int64) kv.Key {
|
|
buf := make([]byte, 0, RecordRowKeyLen)
|
|
buf = appendTableRecordPrefix(buf, tableID)
|
|
buf = codec.EncodeInt(buf, handle)
|
|
return buf
|
|
}
|
|
|
|
// CutRowKeyPrefix cuts the row key prefix.
|
|
func CutRowKeyPrefix(key kv.Key) []byte {
|
|
return key[prefixLen:]
|
|
}
|
|
|
|
// EncodeRecordKey encodes the recordPrefix, row handle into a kv.Key.
|
|
func EncodeRecordKey(recordPrefix kv.Key, h int64) kv.Key {
|
|
buf := make([]byte, 0, len(recordPrefix)+idLen)
|
|
buf = append(buf, recordPrefix...)
|
|
buf = codec.EncodeInt(buf, h)
|
|
return buf
|
|
}
|
|
|
|
func hasTablePrefix(key kv.Key) bool {
|
|
return key[0] == tablePrefix[0]
|
|
}
|
|
|
|
func hasRecordPrefixSep(key kv.Key) bool {
|
|
return key[0] == recordPrefixSep[0] && key[1] == recordPrefixSep[1]
|
|
}
|
|
|
|
// DecodeRecordKey decodes the key and gets the tableID, handle.
|
|
func DecodeRecordKey(key kv.Key) (tableID int64, handle int64, err error) {
|
|
if len(key) <= prefixLen {
|
|
return 0, 0, errInvalidRecordKey.GenWithStack("invalid record key - %q", key)
|
|
}
|
|
|
|
k := key
|
|
if !hasTablePrefix(key) {
|
|
return 0, 0, errInvalidRecordKey.GenWithStack("invalid record key - %q", k)
|
|
}
|
|
|
|
key = key[tablePrefixLength:]
|
|
key, tableID, err = codec.DecodeInt(key)
|
|
if err != nil {
|
|
return 0, 0, errors.Trace(err)
|
|
}
|
|
|
|
if !hasRecordPrefixSep(key) {
|
|
return 0, 0, errInvalidRecordKey.GenWithStack("invalid record key - %q", k)
|
|
}
|
|
|
|
key = key[recordPrefixSepLength:]
|
|
key, handle, err = codec.DecodeInt(key)
|
|
if err != nil {
|
|
return 0, 0, errors.Trace(err)
|
|
}
|
|
return
|
|
}
|
|
|
|
// DecodeIndexKey decodes the key and gets the tableID, indexID, indexValues.
|
|
func DecodeIndexKey(key kv.Key) (tableID int64, indexID int64, indexValues []string, err error) {
|
|
k := key
|
|
|
|
tableID, indexID, key, err = DecodeIndexKeyPrefix(key)
|
|
if err != nil {
|
|
return 0, 0, nil, errors.Trace(err)
|
|
}
|
|
|
|
for len(key) > 0 {
|
|
// FIXME: Without the schema information, we can only decode the raw kind of
|
|
// the column. For instance, MysqlTime is internally saved as uint64.
|
|
remain, d, e := codec.DecodeOne(key)
|
|
if e != nil {
|
|
return 0, 0, nil, errInvalidIndexKey.GenWithStack("invalid index key - %q %v", k, e)
|
|
}
|
|
str, e1 := d.ToString()
|
|
if e1 != nil {
|
|
return 0, 0, nil, errInvalidIndexKey.GenWithStack("invalid index key - %q %v", k, e1)
|
|
}
|
|
indexValues = append(indexValues, str)
|
|
key = remain
|
|
}
|
|
return
|
|
}
|
|
|
|
// DecodeMetaKey decodes the key and get the meta key and meta field.
|
|
func DecodeMetaKey(ek kv.Key) (key []byte, field []byte, err error) {
|
|
var tp uint64
|
|
if !bytes.HasPrefix(ek, metaPrefix) {
|
|
return nil, nil, errors.New("invalid encoded hash data key prefix")
|
|
}
|
|
ek = ek[metaPrefixLength:]
|
|
ek, key, err = codec.DecodeBytes(ek, nil)
|
|
if err != nil {
|
|
return nil, nil, errors.Trace(err)
|
|
}
|
|
ek, tp, err = codec.DecodeUint(ek)
|
|
if err != nil {
|
|
return nil, nil, errors.Trace(err)
|
|
} else if structure.TypeFlag(tp) != structure.HashData {
|
|
return nil, nil, errors.Errorf("invalid encoded hash data key flag %c", byte(tp))
|
|
}
|
|
_, field, err = codec.DecodeBytes(ek, nil)
|
|
return key, field, errors.Trace(err)
|
|
}
|
|
|
|
// DecodeIndexKeyPrefix decodes the key and gets the tableID, indexID, indexValues.
|
|
func DecodeIndexKeyPrefix(key kv.Key) (tableID int64, indexID int64, indexValues []byte, err error) {
|
|
k := key
|
|
|
|
tableID, indexID, isRecord, err := DecodeKeyHead(key)
|
|
if err != nil {
|
|
return 0, 0, nil, errors.Trace(err)
|
|
}
|
|
if isRecord {
|
|
return 0, 0, nil, errInvalidIndexKey.GenWithStack("invalid index key - %q", k)
|
|
}
|
|
indexValues = key[prefixLen+idLen:]
|
|
|
|
return tableID, indexID, indexValues, nil
|
|
}
|
|
|
|
// DecodeKeyHead decodes the key's head and gets the tableID, indexID. isRecordKey is true when is a record key.
|
|
func DecodeKeyHead(key kv.Key) (tableID int64, indexID int64, isRecordKey bool, err error) {
|
|
isRecordKey = false
|
|
k := key
|
|
if !key.HasPrefix(tablePrefix) {
|
|
err = errInvalidKey.GenWithStack("invalid key - %q", k)
|
|
return
|
|
}
|
|
|
|
key = key[len(tablePrefix):]
|
|
key, tableID, err = codec.DecodeInt(key)
|
|
if err != nil {
|
|
err = errors.Trace(err)
|
|
return
|
|
}
|
|
|
|
if key.HasPrefix(recordPrefixSep) {
|
|
isRecordKey = true
|
|
return
|
|
}
|
|
if !key.HasPrefix(indexPrefixSep) {
|
|
err = errInvalidKey.GenWithStack("invalid key - %q", k)
|
|
return
|
|
}
|
|
|
|
key = key[len(indexPrefixSep):]
|
|
|
|
key, indexID, err = codec.DecodeInt(key)
|
|
if err != nil {
|
|
err = errors.Trace(err)
|
|
return
|
|
}
|
|
return
|
|
}
|
|
|
|
// DecodeTableID decodes the table ID of the key, if the key is not table key, returns 0.
|
|
func DecodeTableID(key kv.Key) int64 {
|
|
if !key.HasPrefix(tablePrefix) {
|
|
return 0
|
|
}
|
|
key = key[len(tablePrefix):]
|
|
_, tableID, err := codec.DecodeInt(key)
|
|
// TODO: return error.
|
|
terror.Log(errors.Trace(err))
|
|
return tableID
|
|
}
|
|
|
|
// DecodeRowKey decodes the key and gets the handle.
|
|
func DecodeRowKey(key kv.Key) (int64, error) {
|
|
if len(key) != RecordRowKeyLen || !hasTablePrefix(key) || !hasRecordPrefixSep(key[prefixLen-2:]) {
|
|
return 0, errInvalidKey.GenWithStack("invalid key - %q", key)
|
|
}
|
|
u := binary.BigEndian.Uint64(key[prefixLen:])
|
|
return codec.DecodeCmpUintToInt(u), nil
|
|
}
|
|
|
|
// EncodeValue encodes a go value to bytes.
|
|
func EncodeValue(sc *stmtctx.StatementContext, b []byte, raw types.Datum) ([]byte, error) {
|
|
var v types.Datum
|
|
err := flatten(sc, raw, &v)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return codec.EncodeValue(sc, b, v)
|
|
}
|
|
|
|
// EncodeRow encode row data and column ids into a slice of byte.
|
|
// valBuf and values pass by caller, for reducing EncodeRow allocates temporary bufs. If you pass valBuf and values as nil,
|
|
// EncodeRow will allocate it.
|
|
func EncodeRow(sc *stmtctx.StatementContext, row []types.Datum, colIDs []int64, valBuf []byte, values []types.Datum, e *rowcodec.Encoder) ([]byte, error) {
|
|
if len(row) != len(colIDs) {
|
|
return nil, errors.Errorf("EncodeRow error: data and columnID count not match %d vs %d", len(row), len(colIDs))
|
|
}
|
|
if e.Enable {
|
|
return e.Encode(sc, colIDs, row, valBuf)
|
|
}
|
|
return EncodeOldRow(sc, row, colIDs, valBuf, values)
|
|
}
|
|
|
|
// EncodeOldRow encode row data and column ids into a slice of byte.
|
|
// Row layout: colID1, value1, colID2, value2, .....
|
|
// valBuf and values pass by caller, for reducing EncodeOldRow allocates temporary bufs. If you pass valBuf and values as nil,
|
|
// EncodeOldRow will allocate it.
|
|
func EncodeOldRow(sc *stmtctx.StatementContext, row []types.Datum, colIDs []int64, valBuf []byte, values []types.Datum) ([]byte, error) {
|
|
if len(row) != len(colIDs) {
|
|
return nil, errors.Errorf("EncodeRow error: data and columnID count not match %d vs %d", len(row), len(colIDs))
|
|
}
|
|
valBuf = valBuf[:0]
|
|
if values == nil {
|
|
values = make([]types.Datum, len(row)*2)
|
|
}
|
|
for i, c := range row {
|
|
id := colIDs[i]
|
|
values[2*i].SetInt64(id)
|
|
err := flatten(sc, c, &values[2*i+1])
|
|
if err != nil {
|
|
return valBuf, errors.Trace(err)
|
|
}
|
|
}
|
|
if len(values) == 0 {
|
|
// We could not set nil value into kv.
|
|
return append(valBuf, codec.NilFlag), nil
|
|
}
|
|
return codec.EncodeValue(sc, valBuf, values...)
|
|
}
|
|
|
|
func flatten(sc *stmtctx.StatementContext, data types.Datum, ret *types.Datum) error {
|
|
switch data.Kind() {
|
|
case types.KindMysqlTime:
|
|
// for mysql datetime, timestamp and date type
|
|
t := data.GetMysqlTime()
|
|
if t.Type() == mysql.TypeTimestamp && sc.TimeZone != time.UTC {
|
|
err := t.ConvertTimeZone(sc.TimeZone, time.UTC)
|
|
if err != nil {
|
|
return errors.Trace(err)
|
|
}
|
|
}
|
|
v, err := t.ToPackedUint()
|
|
ret.SetUint64(v)
|
|
return errors.Trace(err)
|
|
case types.KindMysqlDuration:
|
|
// for mysql time type
|
|
ret.SetInt64(int64(data.GetMysqlDuration().Duration))
|
|
return nil
|
|
case types.KindMysqlEnum:
|
|
ret.SetUint64(data.GetMysqlEnum().Value)
|
|
return nil
|
|
case types.KindMysqlSet:
|
|
ret.SetUint64(data.GetMysqlSet().Value)
|
|
return nil
|
|
case types.KindBinaryLiteral, types.KindMysqlBit:
|
|
// We don't need to handle errors here since the literal is ensured to be able to store in uint64 in convertToMysqlBit.
|
|
val, err := data.GetBinaryLiteral().ToInt(sc)
|
|
if err != nil {
|
|
return errors.Trace(err)
|
|
}
|
|
ret.SetUint64(val)
|
|
return nil
|
|
default:
|
|
*ret = data
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// DecodeColumnValue decodes data to a Datum according to the column info.
|
|
func DecodeColumnValue(data []byte, ft *types.FieldType, loc *time.Location) (types.Datum, error) {
|
|
_, d, err := codec.DecodeOne(data)
|
|
if err != nil {
|
|
return types.Datum{}, errors.Trace(err)
|
|
}
|
|
colDatum, err := unflatten(d, ft, loc)
|
|
if err != nil {
|
|
return types.Datum{}, errors.Trace(err)
|
|
}
|
|
return colDatum, nil
|
|
}
|
|
|
|
// DecodeRowWithMapNew decode a row to datum map.
|
|
func DecodeRowWithMapNew(b []byte, cols map[int64]*types.FieldType, loc *time.Location, row map[int64]types.Datum) (map[int64]types.Datum, error) {
|
|
if row == nil {
|
|
row = make(map[int64]types.Datum, len(cols))
|
|
}
|
|
if b == nil {
|
|
return row, nil
|
|
}
|
|
if len(b) == 1 && b[0] == codec.NilFlag {
|
|
return row, nil
|
|
}
|
|
|
|
reqCols := make([]rowcodec.ColInfo, len(cols))
|
|
var idx int
|
|
for id, tp := range cols {
|
|
reqCols[idx] = rowcodec.ColInfo{
|
|
ID: id,
|
|
Tp: int32(tp.Tp),
|
|
Flag: int32(tp.Flag),
|
|
Flen: tp.Flen,
|
|
Decimal: tp.Decimal,
|
|
Elems: tp.Elems,
|
|
Collate: tp.Collate,
|
|
}
|
|
idx++
|
|
}
|
|
// for decodeToMap:
|
|
// - no need handle
|
|
// - no need get default value
|
|
rd := rowcodec.NewDatumMapDecoder(reqCols, -1, loc)
|
|
return rd.DecodeToDatumMap(b, -1, row)
|
|
}
|
|
|
|
// DecodeRowWithMap decodes a byte slice into datums with a existing row map.
|
|
// Row layout: colID1, value1, colID2, value2, .....
|
|
func DecodeRowWithMap(b []byte, cols map[int64]*types.FieldType, loc *time.Location, row map[int64]types.Datum) (map[int64]types.Datum, error) {
|
|
if row == nil {
|
|
row = make(map[int64]types.Datum, len(cols))
|
|
}
|
|
if b == nil {
|
|
return row, nil
|
|
}
|
|
if len(b) == 1 && b[0] == codec.NilFlag {
|
|
return row, nil
|
|
}
|
|
cnt := 0
|
|
var (
|
|
data []byte
|
|
err error
|
|
)
|
|
for len(b) > 0 {
|
|
// Get col id.
|
|
data, b, err = codec.CutOne(b)
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
_, cid, err := codec.DecodeOne(data)
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
// Get col value.
|
|
data, b, err = codec.CutOne(b)
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
id := cid.GetInt64()
|
|
ft, ok := cols[id]
|
|
if ok {
|
|
_, v, err := codec.DecodeOne(data)
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
v, err = unflatten(v, ft, loc)
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
row[id] = v
|
|
cnt++
|
|
if cnt == len(cols) {
|
|
// Get enough data.
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return row, nil
|
|
}
|
|
|
|
// DecodeRow decodes a byte slice into datums.
|
|
// Row layout: colID1, value1, colID2, value2, .....
|
|
func DecodeRow(b []byte, cols map[int64]*types.FieldType, loc *time.Location) (map[int64]types.Datum, error) {
|
|
if !rowcodec.IsNewFormat(b) {
|
|
return DecodeRowWithMap(b, cols, loc, nil)
|
|
}
|
|
return DecodeRowWithMapNew(b, cols, loc, nil)
|
|
}
|
|
|
|
// CutRowNew cuts encoded row into byte slices and return columns' byte slice.
|
|
// Row layout: colID1, value1, colID2, value2, .....
|
|
func CutRowNew(data []byte, colIDs map[int64]int) ([][]byte, error) {
|
|
if data == nil {
|
|
return nil, nil
|
|
}
|
|
if len(data) == 1 && data[0] == codec.NilFlag {
|
|
return nil, nil
|
|
}
|
|
|
|
var (
|
|
cnt int
|
|
b []byte
|
|
err error
|
|
cid int64
|
|
)
|
|
row := make([][]byte, len(colIDs))
|
|
for len(data) > 0 && cnt < len(colIDs) {
|
|
// Get col id.
|
|
data, cid, err = codec.CutColumnID(data)
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
|
|
// Get col value.
|
|
b, data, err = codec.CutOne(data)
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
|
|
offset, ok := colIDs[cid]
|
|
if ok {
|
|
row[offset] = b
|
|
cnt++
|
|
}
|
|
}
|
|
return row, nil
|
|
}
|
|
|
|
// UnflattenDatums converts raw datums to column datums.
|
|
func UnflattenDatums(datums []types.Datum, fts []*types.FieldType, loc *time.Location) ([]types.Datum, error) {
|
|
for i, datum := range datums {
|
|
ft := fts[i]
|
|
uDatum, err := unflatten(datum, ft, loc)
|
|
if err != nil {
|
|
return datums, errors.Trace(err)
|
|
}
|
|
datums[i] = uDatum
|
|
}
|
|
return datums, nil
|
|
}
|
|
|
|
// unflatten converts a raw datum to a column datum.
|
|
func unflatten(datum types.Datum, ft *types.FieldType, loc *time.Location) (types.Datum, error) {
|
|
if datum.IsNull() {
|
|
return datum, nil
|
|
}
|
|
switch ft.Tp {
|
|
case mysql.TypeFloat:
|
|
datum.SetFloat32(float32(datum.GetFloat64()))
|
|
return datum, nil
|
|
case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString:
|
|
datum.SetString(datum.GetString(), ft.Collate)
|
|
case mysql.TypeTiny, mysql.TypeShort, mysql.TypeYear, mysql.TypeInt24,
|
|
mysql.TypeLong, mysql.TypeLonglong, mysql.TypeDouble, mysql.TypeTinyBlob,
|
|
mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob:
|
|
return datum, nil
|
|
case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp:
|
|
t := types.NewTime(types.ZeroCoreTime, ft.Tp, int8(ft.Decimal))
|
|
var err error
|
|
err = t.FromPackedUint(datum.GetUint64())
|
|
if err != nil {
|
|
return datum, errors.Trace(err)
|
|
}
|
|
if ft.Tp == mysql.TypeTimestamp && !t.IsZero() {
|
|
err = t.ConvertTimeZone(time.UTC, loc)
|
|
if err != nil {
|
|
return datum, errors.Trace(err)
|
|
}
|
|
}
|
|
datum.SetUint64(0)
|
|
datum.SetMysqlTime(t)
|
|
return datum, nil
|
|
case mysql.TypeDuration: //duration should read fsp from column meta data
|
|
dur := types.Duration{Duration: time.Duration(datum.GetInt64()), Fsp: int8(ft.Decimal)}
|
|
datum.SetMysqlDuration(dur)
|
|
return datum, nil
|
|
case mysql.TypeEnum:
|
|
// ignore error deliberately, to read empty enum value.
|
|
enum, err := types.ParseEnumValue(ft.Elems, datum.GetUint64())
|
|
if err != nil {
|
|
enum = types.Enum{}
|
|
}
|
|
datum.SetMysqlEnum(enum, ft.Collate)
|
|
return datum, nil
|
|
case mysql.TypeSet:
|
|
set, err := types.ParseSetValue(ft.Elems, datum.GetUint64())
|
|
if err != nil {
|
|
return datum, errors.Trace(err)
|
|
}
|
|
datum.SetMysqlSet(set, ft.Collate)
|
|
return datum, nil
|
|
case mysql.TypeBit:
|
|
val := datum.GetUint64()
|
|
byteSize := (ft.Flen + 7) >> 3
|
|
datum.SetUint64(0)
|
|
datum.SetMysqlBit(types.NewBinaryLiteralFromUint(val, byteSize))
|
|
}
|
|
return datum, nil
|
|
}
|
|
|
|
// EncodeIndexSeekKey encodes an index value to kv.Key.
|
|
func EncodeIndexSeekKey(tableID int64, idxID int64, encodedValue []byte) kv.Key {
|
|
key := make([]byte, 0, prefixLen+len(encodedValue))
|
|
key = appendTableIndexPrefix(key, tableID)
|
|
key = codec.EncodeInt(key, idxID)
|
|
key = append(key, encodedValue...)
|
|
return key
|
|
}
|
|
|
|
// CutIndexKey cuts encoded index key into colIDs to bytes slices map.
|
|
// The returned value b is the remaining bytes of the key which would be empty if it is unique index or handle data
|
|
// if it is non-unique index.
|
|
func CutIndexKey(key kv.Key, colIDs []int64) (values map[int64][]byte, b []byte, err error) {
|
|
b = key[prefixLen+idLen:]
|
|
values = make(map[int64][]byte, len(colIDs))
|
|
for _, id := range colIDs {
|
|
var val []byte
|
|
val, b, err = codec.CutOne(b)
|
|
if err != nil {
|
|
return nil, nil, errors.Trace(err)
|
|
}
|
|
values[id] = val
|
|
}
|
|
return
|
|
}
|
|
|
|
// CutIndexPrefix cuts the index prefix.
|
|
func CutIndexPrefix(key kv.Key) []byte {
|
|
return key[prefixLen+idLen:]
|
|
}
|
|
|
|
// CutIndexKeyNew cuts encoded index key into colIDs to bytes slices.
|
|
// The returned value b is the remaining bytes of the key which would be empty if it is unique index or handle data
|
|
// if it is non-unique index.
|
|
func CutIndexKeyNew(key kv.Key, length int) (values [][]byte, b []byte, err error) {
|
|
b = key[prefixLen+idLen:]
|
|
values = make([][]byte, 0, length)
|
|
for i := 0; i < length; i++ {
|
|
var val []byte
|
|
val, b, err = codec.CutOne(b)
|
|
if err != nil {
|
|
return nil, nil, errors.Trace(err)
|
|
}
|
|
values = append(values, val)
|
|
}
|
|
return
|
|
}
|
|
|
|
// HandleStatus is the handle status in index.
|
|
type HandleStatus int
|
|
|
|
const (
|
|
// HandleNotExists means no need to decode handle value when DecodeIndexKV.
|
|
HandleNotExists HandleStatus = iota
|
|
// HandleIsSigned means decode handle value as int64 when DecodeIndexKV.
|
|
HandleIsSigned
|
|
// HandleIsUnsigned means decode handle value as uint64 when DecodeIndexKV.
|
|
HandleIsUnsigned
|
|
)
|
|
|
|
func handleExists(hdStatus HandleStatus) bool {
|
|
return hdStatus != HandleNotExists
|
|
}
|
|
|
|
// reEncodeHandleByStatus first decode the value into a int or uint decided by the hdStatus, then encode it so that it can
|
|
// be properly decoded.
|
|
func reEncodeHandleByStatus(value []byte, hdStatus HandleStatus) ([]byte, error) {
|
|
handle, err := DecodeIndexValueAsHandle(value)
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
var handleDatum types.Datum
|
|
if hdStatus == HandleIsUnsigned {
|
|
handleDatum = types.NewUintDatum(uint64(handle))
|
|
} else {
|
|
handleDatum = types.NewIntDatum(handle)
|
|
}
|
|
handleBytes := make([]byte, 0, 8)
|
|
handleBytes, err = codec.EncodeValue(nil, handleBytes, handleDatum)
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
return handleBytes, nil
|
|
}
|
|
|
|
func decodeIndexKvNewCollation(key, value []byte, colsLen int, hdStatus HandleStatus, columns []rowcodec.ColInfo) ([][]byte, error) {
|
|
colIDs := make(map[int64]int, len(columns))
|
|
for i, col := range columns {
|
|
colIDs[col.ID] = i
|
|
}
|
|
// We don't need to decode handle here, and colIDs >= 0 always.
|
|
rd := rowcodec.NewByteDecoder(columns, -1, nil, nil)
|
|
vLen := len(value)
|
|
tailLen := int(value[0])
|
|
resultValues, err := rd.DecodeToBytesNoHandle(colIDs, value[1:vLen-tailLen])
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
|
|
if tailLen < 8 {
|
|
// In non-unique index.
|
|
if handleExists(hdStatus) {
|
|
resultValues = append(resultValues, key[len(key)-9:])
|
|
}
|
|
} else {
|
|
// In unique index.
|
|
if handleExists(hdStatus) {
|
|
handleBytes, err := reEncodeHandleByStatus(value[vLen-tailLen:], hdStatus)
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
resultValues = append(resultValues, handleBytes)
|
|
}
|
|
}
|
|
return resultValues, nil
|
|
}
|
|
|
|
func decodeIndexKvOldCollation(key, value []byte, colsLen int, hdStatus HandleStatus) ([][]byte, error) {
|
|
resultValues, b, err := CutIndexKeyNew(key, colsLen)
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
if len(b) > 0 {
|
|
// non-unique index
|
|
if handleExists(hdStatus) {
|
|
resultValues = append(resultValues, b)
|
|
}
|
|
} else if handleExists(hdStatus) {
|
|
// unique index
|
|
handleBytes, err := reEncodeHandleByStatus(value, hdStatus)
|
|
if err != nil {
|
|
return nil, errors.Trace(err)
|
|
}
|
|
resultValues = append(resultValues, handleBytes)
|
|
}
|
|
return resultValues, nil
|
|
}
|
|
|
|
// DecodeIndexKV uses to decode index key values.
|
|
func DecodeIndexKV(key, value []byte, colsLen int, hdStatus HandleStatus, columns []rowcodec.ColInfo) ([][]byte, error) {
|
|
if len(value) > MaxOldEncodeValueLen {
|
|
return decodeIndexKvNewCollation(key, value, colsLen, hdStatus, columns)
|
|
}
|
|
return decodeIndexKvOldCollation(key, value, colsLen, hdStatus)
|
|
}
|
|
|
|
// DecodeIndexHandle uses to decode the handle from index key/value.
|
|
func DecodeIndexHandle(key, value []byte, colsLen int, pkTp *types.FieldType) (int64, error) {
|
|
_, b, err := CutIndexKeyNew(key, colsLen)
|
|
if err != nil {
|
|
return 0, errors.Trace(err)
|
|
}
|
|
if len(b) > 0 {
|
|
d, err := DecodeColumnValue(b, pkTp, nil)
|
|
if err != nil {
|
|
return 0, errors.Trace(err)
|
|
}
|
|
return d.GetInt64(), nil
|
|
|
|
} else if len(value) >= 8 {
|
|
return decodeHandleInIndexValue(value)
|
|
}
|
|
// Should never execute to here.
|
|
return 0, errors.Errorf("no handle in index key: %v, value: %v", key, value)
|
|
}
|
|
|
|
func decodeHandleInIndexValue(value []byte) (int64, error) {
|
|
if len(value) > MaxOldEncodeValueLen {
|
|
tailLen := value[0]
|
|
if tailLen >= 8 {
|
|
return DecodeIndexValueAsHandle(value[len(value)-int(tailLen):])
|
|
}
|
|
// Should never executed to here, since the handle is encoded in IndexKey.
|
|
return 0, errors.Errorf("no handle in index value: %v", value)
|
|
}
|
|
return DecodeIndexValueAsHandle(value)
|
|
}
|
|
|
|
// DecodeIndexValueAsHandle uses to decode index value as handle id.
|
|
func DecodeIndexValueAsHandle(data []byte) (int64, error) {
|
|
var h int64
|
|
buf := bytes.NewBuffer(data)
|
|
err := binary.Read(buf, binary.BigEndian, &h)
|
|
return h, errors.Trace(err)
|
|
}
|
|
|
|
// EncodeTableIndexPrefix encodes index prefix with tableID and idxID.
|
|
func EncodeTableIndexPrefix(tableID, idxID int64) kv.Key {
|
|
key := make([]byte, 0, prefixLen)
|
|
key = appendTableIndexPrefix(key, tableID)
|
|
key = codec.EncodeInt(key, idxID)
|
|
return key
|
|
}
|
|
|
|
// EncodeTablePrefix encodes table prefix with table ID.
|
|
func EncodeTablePrefix(tableID int64) kv.Key {
|
|
var key kv.Key
|
|
key = append(key, tablePrefix...)
|
|
key = codec.EncodeInt(key, tableID)
|
|
return key
|
|
}
|
|
|
|
// appendTableRecordPrefix appends table record prefix "t[tableID]_r".
|
|
func appendTableRecordPrefix(buf []byte, tableID int64) []byte {
|
|
buf = append(buf, tablePrefix...)
|
|
buf = codec.EncodeInt(buf, tableID)
|
|
buf = append(buf, recordPrefixSep...)
|
|
return buf
|
|
}
|
|
|
|
// appendTableIndexPrefix appends table index prefix "t[tableID]_i".
|
|
func appendTableIndexPrefix(buf []byte, tableID int64) []byte {
|
|
buf = append(buf, tablePrefix...)
|
|
buf = codec.EncodeInt(buf, tableID)
|
|
buf = append(buf, indexPrefixSep...)
|
|
return buf
|
|
}
|
|
|
|
// ReplaceRecordKeyTableID replace the tableID in the recordKey buf.
|
|
func ReplaceRecordKeyTableID(buf []byte, tableID int64) []byte {
|
|
if len(buf) < len(tablePrefix)+8 {
|
|
return buf
|
|
}
|
|
|
|
u := codec.EncodeIntToCmpUint(tableID)
|
|
binary.BigEndian.PutUint64(buf[len(tablePrefix):], u)
|
|
return buf
|
|
}
|
|
|
|
// GenTableRecordPrefix composes record prefix with tableID: "t[tableID]_r".
|
|
func GenTableRecordPrefix(tableID int64) kv.Key {
|
|
buf := make([]byte, 0, len(tablePrefix)+8+len(recordPrefixSep))
|
|
return appendTableRecordPrefix(buf, tableID)
|
|
}
|
|
|
|
// GenTableIndexPrefix composes index prefix with tableID: "t[tableID]_i".
|
|
func GenTableIndexPrefix(tableID int64) kv.Key {
|
|
buf := make([]byte, 0, len(tablePrefix)+8+len(indexPrefixSep))
|
|
return appendTableIndexPrefix(buf, tableID)
|
|
}
|
|
|
|
// IsIndexKey is used to check whether the key is an index key.
|
|
func IsIndexKey(k []byte) bool {
|
|
return len(k) > 11 && k[0] == 't' && k[10] == 'i'
|
|
}
|
|
|
|
// IsUntouchedIndexKValue uses to check whether the key is index key, and the value is untouched,
|
|
// since the untouched index key/value is no need to commit.
|
|
func IsUntouchedIndexKValue(k, v []byte) bool {
|
|
if !IsIndexKey(k) {
|
|
return false
|
|
}
|
|
vLen := len(v)
|
|
if vLen <= MaxOldEncodeValueLen {
|
|
return (vLen == 1 || vLen == 9) && v[vLen-1] == kv.UnCommitIndexKVFlag
|
|
}
|
|
// New index value format
|
|
tailLen := int(v[0])
|
|
if tailLen < 8 {
|
|
// Non-unique index.
|
|
return tailLen >= 1 && v[vLen-1] == kv.UnCommitIndexKVFlag
|
|
}
|
|
// Unique index
|
|
return tailLen == 9
|
|
}
|
|
|
|
// GenTablePrefix composes table record and index prefix: "t[tableID]".
|
|
func GenTablePrefix(tableID int64) kv.Key {
|
|
buf := make([]byte, 0, len(tablePrefix)+8)
|
|
buf = append(buf, tablePrefix...)
|
|
buf = codec.EncodeInt(buf, tableID)
|
|
return buf
|
|
}
|
|
|
|
// TruncateToRowKeyLen truncates the key to row key length if the key is longer than row key.
|
|
func TruncateToRowKeyLen(key kv.Key) kv.Key {
|
|
if len(key) > RecordRowKeyLen {
|
|
return key[:RecordRowKeyLen]
|
|
}
|
|
return key
|
|
}
|
|
|
|
// GetTableHandleKeyRange returns table handle's key range with tableID.
|
|
func GetTableHandleKeyRange(tableID int64) (startKey, endKey []byte) {
|
|
startKey = EncodeRowKeyWithHandle(tableID, math.MinInt64)
|
|
endKey = EncodeRowKeyWithHandle(tableID, math.MaxInt64)
|
|
return
|
|
}
|
|
|
|
// GetTableIndexKeyRange returns table index's key range with tableID and indexID.
|
|
func GetTableIndexKeyRange(tableID, indexID int64) (startKey, endKey []byte) {
|
|
startKey = EncodeIndexSeekKey(tableID, indexID, nil)
|
|
endKey = EncodeIndexSeekKey(tableID, indexID, []byte{255})
|
|
return
|
|
}
|
|
|
|
// GetIndexKeyBuf reuse or allocate buffer.
|
|
func GetIndexKeyBuf(buf []byte, defaultCap int) []byte {
|
|
if buf != nil {
|
|
return buf[:0]
|
|
}
|
|
|
|
return make([]byte, 0, defaultCap)
|
|
}
|
|
|
|
// GenIndexKey generates index key using input physical table id.
|
|
func GenIndexKey(sc *stmtctx.StatementContext, tblInfo *model.TableInfo, idxInfo *model.IndexInfo,
|
|
phyTblID int64, indexedValues []types.Datum, h int64, buf []byte) (key []byte, distinct bool, err error) {
|
|
if idxInfo.Unique {
|
|
// See https://dev.mysql.com/doc/refman/5.7/en/create-index.html
|
|
// A UNIQUE index creates a constraint such that all values in the index must be distinct.
|
|
// An error occurs if you try to add a new row with a key value that matches an existing row.
|
|
// For all engines, a UNIQUE index permits multiple NULL values for columns that can contain NULL.
|
|
distinct = true
|
|
for _, cv := range indexedValues {
|
|
if cv.IsNull() {
|
|
distinct = false
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// For string columns, indexes can be created using only the leading part of column values,
|
|
// using col_name(length) syntax to specify an index prefix length.
|
|
indexedValues = TruncateIndexValuesIfNeeded(tblInfo, idxInfo, indexedValues)
|
|
key = GetIndexKeyBuf(buf, RecordRowKeyLen+len(indexedValues)*9+9)
|
|
key = appendTableIndexPrefix(key, phyTblID)
|
|
key = codec.EncodeInt(key, idxInfo.ID)
|
|
key, err = codec.EncodeKey(sc, key, indexedValues...)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
if !distinct {
|
|
key, err = codec.EncodeKey(sc, key, types.NewDatum(h))
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// GenIndexValue creates encoded index value and returns the result.
|
|
func GenIndexValue(sc *stmtctx.StatementContext, tblInfo *model.TableInfo, idxInfo *model.IndexInfo,
|
|
containNonBinaryString bool, distinct bool, untouched bool, indexedValues []types.Datum, h int64) ([]byte, error) {
|
|
var idxVal []byte
|
|
if collate.NewCollationEnabled() && containNonBinaryString {
|
|
colIds := make([]int64, len(idxInfo.Columns))
|
|
for i, col := range idxInfo.Columns {
|
|
colIds[i] = tblInfo.Columns[col.Offset].ID
|
|
}
|
|
rd := rowcodec.Encoder{Enable: true}
|
|
rowRestoredValue, err := rd.Encode(sc, colIds, indexedValues, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
idxVal = make([]byte, 1+len(rowRestoredValue))
|
|
copy(idxVal[1:], rowRestoredValue)
|
|
tailLen := 0
|
|
if distinct {
|
|
// The len of the idxVal is always >= 10 since len (restoredValue) > 0.
|
|
tailLen += 8
|
|
idxVal = append(idxVal, EncodeHandle(h)...)
|
|
} else if len(idxVal) < 10 {
|
|
// Padding the len to 10
|
|
paddingLen := 10 - len(idxVal)
|
|
tailLen += paddingLen
|
|
idxVal = append(idxVal, bytes.Repeat([]byte{0x0}, paddingLen)...)
|
|
}
|
|
if untouched {
|
|
// If index is untouched and fetch here means the key is exists in TiKV, but not in txn mem-buffer,
|
|
// then should also write the untouched index key/value to mem-buffer to make sure the data
|
|
// is consistent with the index in txn mem-buffer.
|
|
tailLen += 1
|
|
idxVal = append(idxVal, kv.UnCommitIndexKVFlag)
|
|
}
|
|
idxVal[0] = byte(tailLen)
|
|
} else {
|
|
idxVal = make([]byte, 0)
|
|
if distinct {
|
|
idxVal = EncodeHandle(h)
|
|
}
|
|
if untouched {
|
|
// If index is untouched and fetch here means the key is exists in TiKV, but not in txn mem-buffer,
|
|
// then should also write the untouched index key/value to mem-buffer to make sure the data
|
|
// is consistent with the index in txn mem-buffer.
|
|
idxVal = append(idxVal, kv.UnCommitIndexKVFlag)
|
|
}
|
|
if len(idxVal) == 0 {
|
|
idxVal = []byte{'0'}
|
|
}
|
|
}
|
|
return idxVal, nil
|
|
}
|
|
|
|
// TruncateIndexValuesIfNeeded truncates the index values created using only the leading part of column values.
|
|
func TruncateIndexValuesIfNeeded(tblInfo *model.TableInfo, idxInfo *model.IndexInfo, indexedValues []types.Datum) []types.Datum {
|
|
for i := 0; i < len(indexedValues); i++ {
|
|
v := &indexedValues[i]
|
|
if v.Kind() == types.KindString || v.Kind() == types.KindBytes {
|
|
ic := idxInfo.Columns[i]
|
|
colCharset := tblInfo.Columns[ic.Offset].Charset
|
|
colValue := v.GetBytes()
|
|
isUTF8Charset := colCharset == charset.CharsetUTF8 || colCharset == charset.CharsetUTF8MB4
|
|
origKind := v.Kind()
|
|
if isUTF8Charset {
|
|
if ic.Length != types.UnspecifiedLength && utf8.RuneCount(colValue) > ic.Length {
|
|
rs := bytes.Runes(colValue)
|
|
truncateStr := string(rs[:ic.Length])
|
|
// truncate value and limit its length
|
|
v.SetString(truncateStr, tblInfo.Columns[ic.Offset].Collate)
|
|
if origKind == types.KindBytes {
|
|
v.SetBytes(v.GetBytes())
|
|
}
|
|
}
|
|
} else if ic.Length != types.UnspecifiedLength && len(colValue) > ic.Length {
|
|
// truncate value and limit its length
|
|
v.SetBytes(colValue[:ic.Length])
|
|
if origKind == types.KindString {
|
|
v.SetString(v.GetString(), tblInfo.Columns[ic.Offset].Collate)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return indexedValues
|
|
}
|
|
|
|
// EncodeHandle encodes handle in data.
|
|
func EncodeHandle(h int64) []byte {
|
|
var data [8]byte
|
|
binary.BigEndian.PutUint64(data[:], uint64(h))
|
|
return data[:]
|
|
}
|
|
|
|
// DecodeHandle decodes handle in data.
|
|
func DecodeHandle(data []byte) (int64, error) {
|
|
dLen := len(data)
|
|
if dLen <= MaxOldEncodeValueLen {
|
|
return int64(binary.BigEndian.Uint64(data)), nil
|
|
}
|
|
return int64(binary.BigEndian.Uint64(data[dLen-int(data[0]):])), nil
|
|
}
|
|
|