// Copyright 2016 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // See the License for the specific language governing permissions and // limitations under the License. package tablecodec import ( "bytes" "encoding/binary" "math" "time" "unicode/utf8" "github.com/pingcap/errors" "github.com/pingcap/parser/charset" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/structure" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/dbterror" "github.com/pingcap/tidb/util/rowcodec" ) var ( errInvalidKey = dbterror.ClassXEval.NewStd(errno.ErrInvalidKey) errInvalidRecordKey = dbterror.ClassXEval.NewStd(errno.ErrInvalidRecordKey) errInvalidIndexKey = dbterror.ClassXEval.NewStd(errno.ErrInvalidIndexKey) ) var ( tablePrefix = []byte{'t'} recordPrefixSep = []byte("_r") indexPrefixSep = []byte("_i") metaPrefix = []byte{'m'} ) const ( idLen = 8 prefixLen = 1 + idLen /*tableID*/ + 2 // RecordRowKeyLen is public for calculating avgerage row size. RecordRowKeyLen = prefixLen + idLen /*handle*/ tablePrefixLength = 1 recordPrefixSepLength = 2 metaPrefixLength = 1 // MaxOldEncodeValueLen is the maximum len of the old encoding of index value. MaxOldEncodeValueLen = 9 ) // TableSplitKeyLen is the length of key 't{table_id}' which is used for table split. const TableSplitKeyLen = 1 + idLen // TablePrefix returns table's prefix 't'. func TablePrefix() []byte { return tablePrefix } // EncodeRowKey encodes the table id and record handle into a kv.Key func EncodeRowKey(tableID int64, encodedHandle []byte) kv.Key { buf := make([]byte, 0, RecordRowKeyLen) buf = appendTableRecordPrefix(buf, tableID) buf = append(buf, encodedHandle...) return buf } // EncodeRowKeyWithHandle encodes the table id, row handle into a kv.Key func EncodeRowKeyWithHandle(tableID int64, handle int64) kv.Key { buf := make([]byte, 0, RecordRowKeyLen) buf = appendTableRecordPrefix(buf, tableID) buf = codec.EncodeInt(buf, handle) return buf } // CutRowKeyPrefix cuts the row key prefix. func CutRowKeyPrefix(key kv.Key) []byte { return key[prefixLen:] } // EncodeRecordKey encodes the recordPrefix, row handle into a kv.Key. func EncodeRecordKey(recordPrefix kv.Key, h int64) kv.Key { buf := make([]byte, 0, len(recordPrefix)+idLen) buf = append(buf, recordPrefix...) buf = codec.EncodeInt(buf, h) return buf } func hasTablePrefix(key kv.Key) bool { return key[0] == tablePrefix[0] } func hasRecordPrefixSep(key kv.Key) bool { return key[0] == recordPrefixSep[0] && key[1] == recordPrefixSep[1] } // DecodeRecordKey decodes the key and gets the tableID, handle. func DecodeRecordKey(key kv.Key) (tableID int64, handle int64, err error) { if len(key) <= prefixLen { return 0, 0, errInvalidRecordKey.GenWithStack("invalid record key - %q", key) } k := key if !hasTablePrefix(key) { return 0, 0, errInvalidRecordKey.GenWithStack("invalid record key - %q", k) } key = key[tablePrefixLength:] key, tableID, err = codec.DecodeInt(key) if err != nil { return 0, 0, errors.Trace(err) } if !hasRecordPrefixSep(key) { return 0, 0, errInvalidRecordKey.GenWithStack("invalid record key - %q", k) } key = key[recordPrefixSepLength:] key, handle, err = codec.DecodeInt(key) if err != nil { return 0, 0, errors.Trace(err) } return } // DecodeIndexKey decodes the key and gets the tableID, indexID, indexValues. func DecodeIndexKey(key kv.Key) (tableID int64, indexID int64, indexValues []string, err error) { k := key tableID, indexID, key, err = DecodeIndexKeyPrefix(key) if err != nil { return 0, 0, nil, errors.Trace(err) } for len(key) > 0 { // FIXME: Without the schema information, we can only decode the raw kind of // the column. For instance, MysqlTime is internally saved as uint64. remain, d, e := codec.DecodeOne(key) if e != nil { return 0, 0, nil, errInvalidIndexKey.GenWithStack("invalid index key - %q %v", k, e) } str, e1 := d.ToString() if e1 != nil { return 0, 0, nil, errInvalidIndexKey.GenWithStack("invalid index key - %q %v", k, e1) } indexValues = append(indexValues, str) key = remain } return } // DecodeMetaKey decodes the key and get the meta key and meta field. func DecodeMetaKey(ek kv.Key) (key []byte, field []byte, err error) { var tp uint64 if !bytes.HasPrefix(ek, metaPrefix) { return nil, nil, errors.New("invalid encoded hash data key prefix") } ek = ek[metaPrefixLength:] ek, key, err = codec.DecodeBytes(ek, nil) if err != nil { return nil, nil, errors.Trace(err) } ek, tp, err = codec.DecodeUint(ek) if err != nil { return nil, nil, errors.Trace(err) } else if structure.TypeFlag(tp) != structure.HashData { return nil, nil, errors.Errorf("invalid encoded hash data key flag %c", byte(tp)) } _, field, err = codec.DecodeBytes(ek, nil) return key, field, errors.Trace(err) } // DecodeIndexKeyPrefix decodes the key and gets the tableID, indexID, indexValues. func DecodeIndexKeyPrefix(key kv.Key) (tableID int64, indexID int64, indexValues []byte, err error) { k := key tableID, indexID, isRecord, err := DecodeKeyHead(key) if err != nil { return 0, 0, nil, errors.Trace(err) } if isRecord { return 0, 0, nil, errInvalidIndexKey.GenWithStack("invalid index key - %q", k) } indexValues = key[prefixLen+idLen:] return tableID, indexID, indexValues, nil } // DecodeKeyHead decodes the key's head and gets the tableID, indexID. isRecordKey is true when is a record key. func DecodeKeyHead(key kv.Key) (tableID int64, indexID int64, isRecordKey bool, err error) { isRecordKey = false k := key if !key.HasPrefix(tablePrefix) { err = errInvalidKey.GenWithStack("invalid key - %q", k) return } key = key[len(tablePrefix):] key, tableID, err = codec.DecodeInt(key) if err != nil { err = errors.Trace(err) return } if key.HasPrefix(recordPrefixSep) { isRecordKey = true return } if !key.HasPrefix(indexPrefixSep) { err = errInvalidKey.GenWithStack("invalid key - %q", k) return } key = key[len(indexPrefixSep):] key, indexID, err = codec.DecodeInt(key) if err != nil { err = errors.Trace(err) return } return } // DecodeTableID decodes the table ID of the key, if the key is not table key, returns 0. func DecodeTableID(key kv.Key) int64 { if !key.HasPrefix(tablePrefix) { return 0 } key = key[len(tablePrefix):] _, tableID, err := codec.DecodeInt(key) // TODO: return error. terror.Log(errors.Trace(err)) return tableID } // DecodeRowKey decodes the key and gets the handle. func DecodeRowKey(key kv.Key) (int64, error) { if len(key) != RecordRowKeyLen || !hasTablePrefix(key) || !hasRecordPrefixSep(key[prefixLen-2:]) { return 0, errInvalidKey.GenWithStack("invalid key - %q", key) } u := binary.BigEndian.Uint64(key[prefixLen:]) return codec.DecodeCmpUintToInt(u), nil } // EncodeValue encodes a go value to bytes. func EncodeValue(sc *stmtctx.StatementContext, b []byte, raw types.Datum) ([]byte, error) { var v types.Datum err := flatten(sc, raw, &v) if err != nil { return nil, err } return codec.EncodeValue(sc, b, v) } // EncodeRow encode row data and column ids into a slice of byte. // valBuf and values pass by caller, for reducing EncodeRow allocates temporary bufs. If you pass valBuf and values as nil, // EncodeRow will allocate it. func EncodeRow(sc *stmtctx.StatementContext, row []types.Datum, colIDs []int64, valBuf []byte, values []types.Datum, e *rowcodec.Encoder) ([]byte, error) { if len(row) != len(colIDs) { return nil, errors.Errorf("EncodeRow error: data and columnID count not match %d vs %d", len(row), len(colIDs)) } if e.Enable { return e.Encode(sc, colIDs, row, valBuf) } return EncodeOldRow(sc, row, colIDs, valBuf, values) } // EncodeOldRow encode row data and column ids into a slice of byte. // Row layout: colID1, value1, colID2, value2, ..... // valBuf and values pass by caller, for reducing EncodeOldRow allocates temporary bufs. If you pass valBuf and values as nil, // EncodeOldRow will allocate it. func EncodeOldRow(sc *stmtctx.StatementContext, row []types.Datum, colIDs []int64, valBuf []byte, values []types.Datum) ([]byte, error) { if len(row) != len(colIDs) { return nil, errors.Errorf("EncodeRow error: data and columnID count not match %d vs %d", len(row), len(colIDs)) } valBuf = valBuf[:0] if values == nil { values = make([]types.Datum, len(row)*2) } for i, c := range row { id := colIDs[i] values[2*i].SetInt64(id) err := flatten(sc, c, &values[2*i+1]) if err != nil { return valBuf, errors.Trace(err) } } if len(values) == 0 { // We could not set nil value into kv. return append(valBuf, codec.NilFlag), nil } return codec.EncodeValue(sc, valBuf, values...) } func flatten(sc *stmtctx.StatementContext, data types.Datum, ret *types.Datum) error { switch data.Kind() { case types.KindMysqlTime: // for mysql datetime, timestamp and date type t := data.GetMysqlTime() if t.Type() == mysql.TypeTimestamp && sc.TimeZone != time.UTC { err := t.ConvertTimeZone(sc.TimeZone, time.UTC) if err != nil { return errors.Trace(err) } } v, err := t.ToPackedUint() ret.SetUint64(v) return errors.Trace(err) case types.KindMysqlDuration: // for mysql time type ret.SetInt64(int64(data.GetMysqlDuration().Duration)) return nil case types.KindMysqlEnum: ret.SetUint64(data.GetMysqlEnum().Value) return nil case types.KindMysqlSet: ret.SetUint64(data.GetMysqlSet().Value) return nil case types.KindBinaryLiteral, types.KindMysqlBit: // We don't need to handle errors here since the literal is ensured to be able to store in uint64 in convertToMysqlBit. val, err := data.GetBinaryLiteral().ToInt(sc) if err != nil { return errors.Trace(err) } ret.SetUint64(val) return nil default: *ret = data return nil } } // DecodeColumnValue decodes data to a Datum according to the column info. func DecodeColumnValue(data []byte, ft *types.FieldType, loc *time.Location) (types.Datum, error) { _, d, err := codec.DecodeOne(data) if err != nil { return types.Datum{}, errors.Trace(err) } colDatum, err := unflatten(d, ft, loc) if err != nil { return types.Datum{}, errors.Trace(err) } return colDatum, nil } // DecodeRowWithMapNew decode a row to datum map. func DecodeRowWithMapNew(b []byte, cols map[int64]*types.FieldType, loc *time.Location, row map[int64]types.Datum) (map[int64]types.Datum, error) { if row == nil { row = make(map[int64]types.Datum, len(cols)) } if b == nil { return row, nil } if len(b) == 1 && b[0] == codec.NilFlag { return row, nil } reqCols := make([]rowcodec.ColInfo, len(cols)) var idx int for id, tp := range cols { reqCols[idx] = rowcodec.ColInfo{ ID: id, Tp: int32(tp.Tp), Flag: int32(tp.Flag), Flen: tp.Flen, Decimal: tp.Decimal, Elems: tp.Elems, Collate: tp.Collate, } idx++ } // for decodeToMap: // - no need handle // - no need get default value rd := rowcodec.NewDatumMapDecoder(reqCols, -1, loc) return rd.DecodeToDatumMap(b, -1, row) } // DecodeRowWithMap decodes a byte slice into datums with a existing row map. // Row layout: colID1, value1, colID2, value2, ..... func DecodeRowWithMap(b []byte, cols map[int64]*types.FieldType, loc *time.Location, row map[int64]types.Datum) (map[int64]types.Datum, error) { if row == nil { row = make(map[int64]types.Datum, len(cols)) } if b == nil { return row, nil } if len(b) == 1 && b[0] == codec.NilFlag { return row, nil } cnt := 0 var ( data []byte err error ) for len(b) > 0 { // Get col id. data, b, err = codec.CutOne(b) if err != nil { return nil, errors.Trace(err) } _, cid, err := codec.DecodeOne(data) if err != nil { return nil, errors.Trace(err) } // Get col value. data, b, err = codec.CutOne(b) if err != nil { return nil, errors.Trace(err) } id := cid.GetInt64() ft, ok := cols[id] if ok { _, v, err := codec.DecodeOne(data) if err != nil { return nil, errors.Trace(err) } v, err = unflatten(v, ft, loc) if err != nil { return nil, errors.Trace(err) } row[id] = v cnt++ if cnt == len(cols) { // Get enough data. break } } } return row, nil } // DecodeRow decodes a byte slice into datums. // Row layout: colID1, value1, colID2, value2, ..... func DecodeRow(b []byte, cols map[int64]*types.FieldType, loc *time.Location) (map[int64]types.Datum, error) { if !rowcodec.IsNewFormat(b) { return DecodeRowWithMap(b, cols, loc, nil) } return DecodeRowWithMapNew(b, cols, loc, nil) } // CutRowNew cuts encoded row into byte slices and return columns' byte slice. // Row layout: colID1, value1, colID2, value2, ..... func CutRowNew(data []byte, colIDs map[int64]int) ([][]byte, error) { if data == nil { return nil, nil } if len(data) == 1 && data[0] == codec.NilFlag { return nil, nil } var ( cnt int b []byte err error cid int64 ) row := make([][]byte, len(colIDs)) for len(data) > 0 && cnt < len(colIDs) { // Get col id. data, cid, err = codec.CutColumnID(data) if err != nil { return nil, errors.Trace(err) } // Get col value. b, data, err = codec.CutOne(data) if err != nil { return nil, errors.Trace(err) } offset, ok := colIDs[cid] if ok { row[offset] = b cnt++ } } return row, nil } // UnflattenDatums converts raw datums to column datums. func UnflattenDatums(datums []types.Datum, fts []*types.FieldType, loc *time.Location) ([]types.Datum, error) { for i, datum := range datums { ft := fts[i] uDatum, err := unflatten(datum, ft, loc) if err != nil { return datums, errors.Trace(err) } datums[i] = uDatum } return datums, nil } // unflatten converts a raw datum to a column datum. func unflatten(datum types.Datum, ft *types.FieldType, loc *time.Location) (types.Datum, error) { if datum.IsNull() { return datum, nil } switch ft.Tp { case mysql.TypeFloat: datum.SetFloat32(float32(datum.GetFloat64())) return datum, nil case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString: datum.SetString(datum.GetString(), ft.Collate) case mysql.TypeTiny, mysql.TypeShort, mysql.TypeYear, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeDouble, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob: return datum, nil case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp: t := types.NewTime(types.ZeroCoreTime, ft.Tp, int8(ft.Decimal)) var err error err = t.FromPackedUint(datum.GetUint64()) if err != nil { return datum, errors.Trace(err) } if ft.Tp == mysql.TypeTimestamp && !t.IsZero() { err = t.ConvertTimeZone(time.UTC, loc) if err != nil { return datum, errors.Trace(err) } } datum.SetUint64(0) datum.SetMysqlTime(t) return datum, nil case mysql.TypeDuration: //duration should read fsp from column meta data dur := types.Duration{Duration: time.Duration(datum.GetInt64()), Fsp: int8(ft.Decimal)} datum.SetMysqlDuration(dur) return datum, nil case mysql.TypeEnum: // ignore error deliberately, to read empty enum value. enum, err := types.ParseEnumValue(ft.Elems, datum.GetUint64()) if err != nil { enum = types.Enum{} } datum.SetMysqlEnum(enum, ft.Collate) return datum, nil case mysql.TypeSet: set, err := types.ParseSetValue(ft.Elems, datum.GetUint64()) if err != nil { return datum, errors.Trace(err) } datum.SetMysqlSet(set, ft.Collate) return datum, nil case mysql.TypeBit: val := datum.GetUint64() byteSize := (ft.Flen + 7) >> 3 datum.SetUint64(0) datum.SetMysqlBit(types.NewBinaryLiteralFromUint(val, byteSize)) } return datum, nil } // EncodeIndexSeekKey encodes an index value to kv.Key. func EncodeIndexSeekKey(tableID int64, idxID int64, encodedValue []byte) kv.Key { key := make([]byte, 0, prefixLen+len(encodedValue)) key = appendTableIndexPrefix(key, tableID) key = codec.EncodeInt(key, idxID) key = append(key, encodedValue...) return key } // CutIndexKey cuts encoded index key into colIDs to bytes slices map. // The returned value b is the remaining bytes of the key which would be empty if it is unique index or handle data // if it is non-unique index. func CutIndexKey(key kv.Key, colIDs []int64) (values map[int64][]byte, b []byte, err error) { b = key[prefixLen+idLen:] values = make(map[int64][]byte, len(colIDs)) for _, id := range colIDs { var val []byte val, b, err = codec.CutOne(b) if err != nil { return nil, nil, errors.Trace(err) } values[id] = val } return } // CutIndexPrefix cuts the index prefix. func CutIndexPrefix(key kv.Key) []byte { return key[prefixLen+idLen:] } // CutIndexKeyNew cuts encoded index key into colIDs to bytes slices. // The returned value b is the remaining bytes of the key which would be empty if it is unique index or handle data // if it is non-unique index. func CutIndexKeyNew(key kv.Key, length int) (values [][]byte, b []byte, err error) { b = key[prefixLen+idLen:] values = make([][]byte, 0, length) for i := 0; i < length; i++ { var val []byte val, b, err = codec.CutOne(b) if err != nil { return nil, nil, errors.Trace(err) } values = append(values, val) } return } // HandleStatus is the handle status in index. type HandleStatus int const ( // HandleNotExists means no need to decode handle value when DecodeIndexKV. HandleNotExists HandleStatus = iota // HandleIsSigned means decode handle value as int64 when DecodeIndexKV. HandleIsSigned // HandleIsUnsigned means decode handle value as uint64 when DecodeIndexKV. HandleIsUnsigned ) func handleExists(hdStatus HandleStatus) bool { return hdStatus != HandleNotExists } // reEncodeHandleByStatus first decode the value into a int or uint decided by the hdStatus, then encode it so that it can // be properly decoded. func reEncodeHandleByStatus(value []byte, hdStatus HandleStatus) ([]byte, error) { handle, err := DecodeIndexValueAsHandle(value) if err != nil { return nil, errors.Trace(err) } var handleDatum types.Datum if hdStatus == HandleIsUnsigned { handleDatum = types.NewUintDatum(uint64(handle)) } else { handleDatum = types.NewIntDatum(handle) } handleBytes := make([]byte, 0, 8) handleBytes, err = codec.EncodeValue(nil, handleBytes, handleDatum) if err != nil { return nil, errors.Trace(err) } return handleBytes, nil } func decodeIndexKvNewCollation(key, value []byte, colsLen int, hdStatus HandleStatus, columns []rowcodec.ColInfo) ([][]byte, error) { colIDs := make(map[int64]int, len(columns)) for i, col := range columns { colIDs[col.ID] = i } // We don't need to decode handle here, and colIDs >= 0 always. rd := rowcodec.NewByteDecoder(columns, -1, nil, nil) vLen := len(value) tailLen := int(value[0]) resultValues, err := rd.DecodeToBytesNoHandle(colIDs, value[1:vLen-tailLen]) if err != nil { return nil, errors.Trace(err) } if tailLen < 8 { // In non-unique index. if handleExists(hdStatus) { resultValues = append(resultValues, key[len(key)-9:]) } } else { // In unique index. if handleExists(hdStatus) { handleBytes, err := reEncodeHandleByStatus(value[vLen-tailLen:], hdStatus) if err != nil { return nil, errors.Trace(err) } resultValues = append(resultValues, handleBytes) } } return resultValues, nil } func decodeIndexKvOldCollation(key, value []byte, colsLen int, hdStatus HandleStatus) ([][]byte, error) { resultValues, b, err := CutIndexKeyNew(key, colsLen) if err != nil { return nil, errors.Trace(err) } if len(b) > 0 { // non-unique index if handleExists(hdStatus) { resultValues = append(resultValues, b) } } else if handleExists(hdStatus) { // unique index handleBytes, err := reEncodeHandleByStatus(value, hdStatus) if err != nil { return nil, errors.Trace(err) } resultValues = append(resultValues, handleBytes) } return resultValues, nil } // DecodeIndexKV uses to decode index key values. func DecodeIndexKV(key, value []byte, colsLen int, hdStatus HandleStatus, columns []rowcodec.ColInfo) ([][]byte, error) { if len(value) > MaxOldEncodeValueLen { return decodeIndexKvNewCollation(key, value, colsLen, hdStatus, columns) } return decodeIndexKvOldCollation(key, value, colsLen, hdStatus) } // DecodeIndexHandle uses to decode the handle from index key/value. func DecodeIndexHandle(key, value []byte, colsLen int, pkTp *types.FieldType) (int64, error) { _, b, err := CutIndexKeyNew(key, colsLen) if err != nil { return 0, errors.Trace(err) } if len(b) > 0 { d, err := DecodeColumnValue(b, pkTp, nil) if err != nil { return 0, errors.Trace(err) } return d.GetInt64(), nil } else if len(value) >= 8 { return decodeHandleInIndexValue(value) } // Should never execute to here. return 0, errors.Errorf("no handle in index key: %v, value: %v", key, value) } func decodeHandleInIndexValue(value []byte) (int64, error) { if len(value) > MaxOldEncodeValueLen { tailLen := value[0] if tailLen >= 8 { return DecodeIndexValueAsHandle(value[len(value)-int(tailLen):]) } // Should never executed to here, since the handle is encoded in IndexKey. return 0, errors.Errorf("no handle in index value: %v", value) } return DecodeIndexValueAsHandle(value) } // DecodeIndexValueAsHandle uses to decode index value as handle id. func DecodeIndexValueAsHandle(data []byte) (int64, error) { var h int64 buf := bytes.NewBuffer(data) err := binary.Read(buf, binary.BigEndian, &h) return h, errors.Trace(err) } // EncodeTableIndexPrefix encodes index prefix with tableID and idxID. func EncodeTableIndexPrefix(tableID, idxID int64) kv.Key { key := make([]byte, 0, prefixLen) key = appendTableIndexPrefix(key, tableID) key = codec.EncodeInt(key, idxID) return key } // EncodeTablePrefix encodes table prefix with table ID. func EncodeTablePrefix(tableID int64) kv.Key { var key kv.Key key = append(key, tablePrefix...) key = codec.EncodeInt(key, tableID) return key } // appendTableRecordPrefix appends table record prefix "t[tableID]_r". func appendTableRecordPrefix(buf []byte, tableID int64) []byte { buf = append(buf, tablePrefix...) buf = codec.EncodeInt(buf, tableID) buf = append(buf, recordPrefixSep...) return buf } // appendTableIndexPrefix appends table index prefix "t[tableID]_i". func appendTableIndexPrefix(buf []byte, tableID int64) []byte { buf = append(buf, tablePrefix...) buf = codec.EncodeInt(buf, tableID) buf = append(buf, indexPrefixSep...) return buf } // ReplaceRecordKeyTableID replace the tableID in the recordKey buf. func ReplaceRecordKeyTableID(buf []byte, tableID int64) []byte { if len(buf) < len(tablePrefix)+8 { return buf } u := codec.EncodeIntToCmpUint(tableID) binary.BigEndian.PutUint64(buf[len(tablePrefix):], u) return buf } // GenTableRecordPrefix composes record prefix with tableID: "t[tableID]_r". func GenTableRecordPrefix(tableID int64) kv.Key { buf := make([]byte, 0, len(tablePrefix)+8+len(recordPrefixSep)) return appendTableRecordPrefix(buf, tableID) } // GenTableIndexPrefix composes index prefix with tableID: "t[tableID]_i". func GenTableIndexPrefix(tableID int64) kv.Key { buf := make([]byte, 0, len(tablePrefix)+8+len(indexPrefixSep)) return appendTableIndexPrefix(buf, tableID) } // IsIndexKey is used to check whether the key is an index key. func IsIndexKey(k []byte) bool { return len(k) > 11 && k[0] == 't' && k[10] == 'i' } // IsUntouchedIndexKValue uses to check whether the key is index key, and the value is untouched, // since the untouched index key/value is no need to commit. func IsUntouchedIndexKValue(k, v []byte) bool { if !IsIndexKey(k) { return false } vLen := len(v) if vLen <= MaxOldEncodeValueLen { return (vLen == 1 || vLen == 9) && v[vLen-1] == kv.UnCommitIndexKVFlag } // New index value format tailLen := int(v[0]) if tailLen < 8 { // Non-unique index. return tailLen >= 1 && v[vLen-1] == kv.UnCommitIndexKVFlag } // Unique index return tailLen == 9 } // GenTablePrefix composes table record and index prefix: "t[tableID]". func GenTablePrefix(tableID int64) kv.Key { buf := make([]byte, 0, len(tablePrefix)+8) buf = append(buf, tablePrefix...) buf = codec.EncodeInt(buf, tableID) return buf } // TruncateToRowKeyLen truncates the key to row key length if the key is longer than row key. func TruncateToRowKeyLen(key kv.Key) kv.Key { if len(key) > RecordRowKeyLen { return key[:RecordRowKeyLen] } return key } // GetTableHandleKeyRange returns table handle's key range with tableID. func GetTableHandleKeyRange(tableID int64) (startKey, endKey []byte) { startKey = EncodeRowKeyWithHandle(tableID, math.MinInt64) endKey = EncodeRowKeyWithHandle(tableID, math.MaxInt64) return } // GetTableIndexKeyRange returns table index's key range with tableID and indexID. func GetTableIndexKeyRange(tableID, indexID int64) (startKey, endKey []byte) { startKey = EncodeIndexSeekKey(tableID, indexID, nil) endKey = EncodeIndexSeekKey(tableID, indexID, []byte{255}) return } // GetIndexKeyBuf reuse or allocate buffer. func GetIndexKeyBuf(buf []byte, defaultCap int) []byte { if buf != nil { return buf[:0] } return make([]byte, 0, defaultCap) } // GenIndexKey generates index key using input physical table id. func GenIndexKey(sc *stmtctx.StatementContext, tblInfo *model.TableInfo, idxInfo *model.IndexInfo, phyTblID int64, indexedValues []types.Datum, h int64, buf []byte) (key []byte, distinct bool, err error) { if idxInfo.Unique { // See https://dev.mysql.com/doc/refman/5.7/en/create-index.html // A UNIQUE index creates a constraint such that all values in the index must be distinct. // An error occurs if you try to add a new row with a key value that matches an existing row. // For all engines, a UNIQUE index permits multiple NULL values for columns that can contain NULL. distinct = true for _, cv := range indexedValues { if cv.IsNull() { distinct = false break } } } // For string columns, indexes can be created using only the leading part of column values, // using col_name(length) syntax to specify an index prefix length. indexedValues = TruncateIndexValuesIfNeeded(tblInfo, idxInfo, indexedValues) key = GetIndexKeyBuf(buf, RecordRowKeyLen+len(indexedValues)*9+9) key = appendTableIndexPrefix(key, phyTblID) key = codec.EncodeInt(key, idxInfo.ID) key, err = codec.EncodeKey(sc, key, indexedValues...) if err != nil { return nil, false, err } if !distinct { key, err = codec.EncodeKey(sc, key, types.NewDatum(h)) if err != nil { return nil, false, err } } return } // GenIndexValue creates encoded index value and returns the result. func GenIndexValue(sc *stmtctx.StatementContext, tblInfo *model.TableInfo, idxInfo *model.IndexInfo, containNonBinaryString bool, distinct bool, untouched bool, indexedValues []types.Datum, h int64) ([]byte, error) { var idxVal []byte if collate.NewCollationEnabled() && containNonBinaryString { colIds := make([]int64, len(idxInfo.Columns)) for i, col := range idxInfo.Columns { colIds[i] = tblInfo.Columns[col.Offset].ID } rd := rowcodec.Encoder{Enable: true} rowRestoredValue, err := rd.Encode(sc, colIds, indexedValues, nil) if err != nil { return nil, err } idxVal = make([]byte, 1+len(rowRestoredValue)) copy(idxVal[1:], rowRestoredValue) tailLen := 0 if distinct { // The len of the idxVal is always >= 10 since len (restoredValue) > 0. tailLen += 8 idxVal = append(idxVal, EncodeHandle(h)...) } else if len(idxVal) < 10 { // Padding the len to 10 paddingLen := 10 - len(idxVal) tailLen += paddingLen idxVal = append(idxVal, bytes.Repeat([]byte{0x0}, paddingLen)...) } if untouched { // If index is untouched and fetch here means the key is exists in TiKV, but not in txn mem-buffer, // then should also write the untouched index key/value to mem-buffer to make sure the data // is consistent with the index in txn mem-buffer. tailLen += 1 idxVal = append(idxVal, kv.UnCommitIndexKVFlag) } idxVal[0] = byte(tailLen) } else { idxVal = make([]byte, 0) if distinct { idxVal = EncodeHandle(h) } if untouched { // If index is untouched and fetch here means the key is exists in TiKV, but not in txn mem-buffer, // then should also write the untouched index key/value to mem-buffer to make sure the data // is consistent with the index in txn mem-buffer. idxVal = append(idxVal, kv.UnCommitIndexKVFlag) } if len(idxVal) == 0 { idxVal = []byte{'0'} } } return idxVal, nil } // TruncateIndexValuesIfNeeded truncates the index values created using only the leading part of column values. func TruncateIndexValuesIfNeeded(tblInfo *model.TableInfo, idxInfo *model.IndexInfo, indexedValues []types.Datum) []types.Datum { for i := 0; i < len(indexedValues); i++ { v := &indexedValues[i] if v.Kind() == types.KindString || v.Kind() == types.KindBytes { ic := idxInfo.Columns[i] colCharset := tblInfo.Columns[ic.Offset].Charset colValue := v.GetBytes() isUTF8Charset := colCharset == charset.CharsetUTF8 || colCharset == charset.CharsetUTF8MB4 origKind := v.Kind() if isUTF8Charset { if ic.Length != types.UnspecifiedLength && utf8.RuneCount(colValue) > ic.Length { rs := bytes.Runes(colValue) truncateStr := string(rs[:ic.Length]) // truncate value and limit its length v.SetString(truncateStr, tblInfo.Columns[ic.Offset].Collate) if origKind == types.KindBytes { v.SetBytes(v.GetBytes()) } } } else if ic.Length != types.UnspecifiedLength && len(colValue) > ic.Length { // truncate value and limit its length v.SetBytes(colValue[:ic.Length]) if origKind == types.KindString { v.SetString(v.GetString(), tblInfo.Columns[ic.Offset].Collate) } } } } return indexedValues } // EncodeHandle encodes handle in data. func EncodeHandle(h int64) []byte { var data [8]byte binary.BigEndian.PutUint64(data[:], uint64(h)) return data[:] } // DecodeHandle decodes handle in data. func DecodeHandle(data []byte) (int64, error) { dLen := len(data) if dLen <= MaxOldEncodeValueLen { return int64(binary.BigEndian.Uint64(data)), nil } return int64(binary.BigEndian.Uint64(data[dLen-int(data[0]):])), nil }