You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

393 lines
12 KiB

// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package owner
import (
"context"
"fmt"
"math"
"os"
"strconv"
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/parser/terror"
"github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/logutil"
"go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/clientv3/concurrency"
"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
"go.etcd.io/etcd/mvcc/mvccpb"
"go.uber.org/zap"
"google.golang.org/grpc"
)
const (
newSessionRetryInterval = 200 * time.Millisecond
logIntervalCnt = int(3 * time.Second / newSessionRetryInterval)
)
// Manager is used to campaign the owner and manage the owner information.
type Manager interface {
// ID returns the ID of the manager.
ID() string
// IsOwner returns whether the ownerManager is the owner.
IsOwner() bool
// RetireOwner make the manager to be a not owner. It's exported for testing.
RetireOwner()
// GetOwnerID gets the owner ID.
GetOwnerID(ctx context.Context) (string, error)
// CampaignOwner campaigns the owner.
CampaignOwner() error
// ResignOwner lets the owner start a new election.
ResignOwner(ctx context.Context) error
// Cancel cancels this etcd ownerManager campaign.
Cancel()
}
const (
// NewSessionDefaultRetryCnt is the default retry times when create new session.
NewSessionDefaultRetryCnt = 3
// NewSessionRetryUnlimited is the unlimited retry times when create new session.
NewSessionRetryUnlimited = math.MaxInt64
keyOpDefaultTimeout = 5 * time.Second
)
// DDLOwnerChecker is used to check whether tidb is owner.
type DDLOwnerChecker interface {
// IsOwner returns whether the ownerManager is the owner.
IsOwner() bool
}
// ownerManager represents the structure which is used for electing owner.
type ownerManager struct {
id string // id is the ID of the manager.
key string
ctx context.Context
prompt string
logPrefix string
logCtx context.Context
etcdCli *clientv3.Client
cancel context.CancelFunc
elec unsafe.Pointer
wg sync.WaitGroup
}
// NewOwnerManager creates a new Manager.
func NewOwnerManager(ctx context.Context, etcdCli *clientv3.Client, prompt, id, key string) Manager {
logPrefix := fmt.Sprintf("[%s] %s ownerManager %s", prompt, key, id)
ctx, cancelFunc := context.WithCancel(ctx)
return &ownerManager{
etcdCli: etcdCli,
id: id,
key: key,
ctx: ctx,
prompt: prompt,
cancel: cancelFunc,
logPrefix: logPrefix,
logCtx: logutil.WithKeyValue(context.Background(), "owner info", logPrefix),
}
}
// ID implements Manager.ID interface.
func (m *ownerManager) ID() string {
return m.id
}
// IsOwner implements Manager.IsOwner interface.
func (m *ownerManager) IsOwner() bool {
return atomic.LoadPointer(&m.elec) != unsafe.Pointer(nil)
}
// Cancel implements Manager.Cancel interface.
func (m *ownerManager) Cancel() {
m.cancel()
m.wg.Wait()
}
// ManagerSessionTTL is the etcd session's TTL in seconds. It's exported for testing.
var ManagerSessionTTL = 60
// setManagerSessionTTL sets the ManagerSessionTTL value, it's used for testing.
func setManagerSessionTTL() error {
ttlStr := os.Getenv("tidb_manager_ttl")
if len(ttlStr) == 0 {
return nil
}
ttl, err := strconv.Atoi(ttlStr)
if err != nil {
return errors.Trace(err)
}
ManagerSessionTTL = ttl
return nil
}
// NewSession creates a new etcd session.
func NewSession(ctx context.Context, logPrefix string, etcdCli *clientv3.Client, retryCnt, ttl int) (*concurrency.Session, error) {
var err error
var etcdSession *concurrency.Session
failedCnt := 0
for i := 0; i < retryCnt; i++ {
if err = contextDone(ctx, err); err != nil {
return etcdSession, errors.Trace(err)
}
failpoint.Inject("closeClient", func(val failpoint.Value) {
if val.(bool) {
if err := etcdCli.Close(); err != nil {
failpoint.Return(etcdSession, errors.Trace(err))
}
}
})
failpoint.Inject("closeGrpc", func(val failpoint.Value) {
if val.(bool) {
if err := etcdCli.ActiveConnection().Close(); err != nil {
failpoint.Return(etcdSession, errors.Trace(err))
}
}
})
startTime := time.Now()
etcdSession, err = concurrency.NewSession(etcdCli,
concurrency.WithTTL(ttl), concurrency.WithContext(ctx))
metrics.NewSessionHistogram.WithLabelValues(logPrefix, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
if err == nil {
break
}
if failedCnt%logIntervalCnt == 0 {
logutil.BgLogger().Warn("failed to new session to etcd", zap.String("ownerInfo", logPrefix), zap.Error(err))
}
time.Sleep(newSessionRetryInterval)
failedCnt++
}
return etcdSession, errors.Trace(err)
}
// CampaignOwner implements Manager.CampaignOwner interface.
func (m *ownerManager) CampaignOwner() error {
logPrefix := fmt.Sprintf("[%s] %s", m.prompt, m.key)
logutil.BgLogger().Info("start campaign owner", zap.String("ownerInfo", logPrefix))
session, err := NewSession(m.ctx, logPrefix, m.etcdCli, NewSessionDefaultRetryCnt, ManagerSessionTTL)
if err != nil {
return errors.Trace(err)
}
m.wg.Add(1)
go m.campaignLoop(session)
return nil
}
// ResignOwner lets the owner start a new election.
func (m *ownerManager) ResignOwner(ctx context.Context) error {
elec := (*concurrency.Election)(atomic.LoadPointer(&m.elec))
if elec == nil {
return errors.Errorf("This node is not a ddl owner, can't be resigned.")
}
childCtx, cancel := context.WithTimeout(ctx, keyOpDefaultTimeout)
err := elec.Resign(childCtx)
cancel()
if err != nil {
return errors.Trace(err)
}
logutil.Logger(m.logCtx).Warn("resign ddl owner success")
return nil
}
func (m *ownerManager) toBeOwner(elec *concurrency.Election) {
atomic.StorePointer(&m.elec, unsafe.Pointer(elec))
}
// RetireOwner make the manager to be a not owner.
func (m *ownerManager) RetireOwner() {
atomic.StorePointer(&m.elec, nil)
}
func (m *ownerManager) campaignLoop(etcdSession *concurrency.Session) {
var cancel context.CancelFunc
ctx, cancel := context.WithCancel(m.ctx)
defer func() {
cancel()
if r := recover(); r != nil {
buf := util.GetStack()
logutil.BgLogger().Error("recover panic", zap.String("prompt", m.prompt), zap.Any("error", r), zap.String("buffer", string(buf)))
metrics.PanicCounter.WithLabelValues(metrics.LabelDDLOwner).Inc()
}
m.wg.Done()
}()
logPrefix := m.logPrefix
logCtx := m.logCtx
var err error
for {
if err != nil {
metrics.CampaignOwnerCounter.WithLabelValues(m.prompt, err.Error()).Inc()
}
select {
case <-etcdSession.Done():
logutil.Logger(logCtx).Info("etcd session is done, creates a new one")
leaseID := etcdSession.Lease()
etcdSession, err = NewSession(ctx, logPrefix, m.etcdCli, NewSessionRetryUnlimited, ManagerSessionTTL)
if err != nil {
logutil.Logger(logCtx).Info("break campaign loop, NewSession failed", zap.Error(err))
m.revokeSession(logPrefix, leaseID)
return
}
case <-ctx.Done():
logutil.Logger(logCtx).Info("break campaign loop, context is done")
m.revokeSession(logPrefix, etcdSession.Lease())
return
default:
}
// If the etcd server turns clocks forward,the following case may occur.
// The etcd server deletes this session's lease ID, but etcd session doesn't find it.
// In this time if we do the campaign operation, the etcd server will return ErrLeaseNotFound.
if terror.ErrorEqual(err, rpctypes.ErrLeaseNotFound) {
if etcdSession != nil {
err = etcdSession.Close()
logutil.Logger(logCtx).Info("etcd session encounters the error of lease not found, closes it", zap.Error(err))
}
continue
}
elec := concurrency.NewElection(etcdSession, m.key)
err = elec.Campaign(ctx, m.id)
if err != nil {
logutil.Logger(logCtx).Info("failed to campaign", zap.Error(err))
continue
}
ownerKey, err := GetOwnerInfo(ctx, logCtx, elec, m.id)
if err != nil {
continue
}
m.toBeOwner(elec)
m.watchOwner(ctx, etcdSession, ownerKey)
m.RetireOwner()
metrics.CampaignOwnerCounter.WithLabelValues(m.prompt, metrics.NoLongerOwner).Inc()
logutil.Logger(logCtx).Warn("is not the owner")
}
}
func (m *ownerManager) revokeSession(logPrefix string, leaseID clientv3.LeaseID) {
// Revoke the session lease.
// If revoke takes longer than the ttl, lease is expired anyway.
cancelCtx, cancel := context.WithTimeout(context.Background(),
time.Duration(ManagerSessionTTL)*time.Second)
_, err := m.etcdCli.Revoke(cancelCtx, leaseID)
cancel()
logutil.Logger(m.logCtx).Info("revoke session", zap.Error(err))
}
// GetOwnerID implements Manager.GetOwnerID interface.
func (m *ownerManager) GetOwnerID(ctx context.Context) (string, error) {
resp, err := m.etcdCli.Get(ctx, m.key, clientv3.WithFirstCreate()...)
if err != nil {
return "", errors.Trace(err)
}
if len(resp.Kvs) == 0 {
return "", concurrency.ErrElectionNoLeader
}
return string(resp.Kvs[0].Value), nil
}
// GetOwnerInfo gets the owner information.
func GetOwnerInfo(ctx, logCtx context.Context, elec *concurrency.Election, id string) (string, error) {
resp, err := elec.Leader(ctx)
if err != nil {
// If no leader elected currently, it returns ErrElectionNoLeader.
logutil.Logger(logCtx).Info("failed to get leader", zap.Error(err))
return "", errors.Trace(err)
}
ownerID := string(resp.Kvs[0].Value)
logutil.Logger(logCtx).Info("get owner", zap.String("ownerID", ownerID))
if ownerID != id {
logutil.Logger(logCtx).Warn("is not the owner")
return "", errors.New("ownerInfoNotMatch")
}
return string(resp.Kvs[0].Key), nil
}
func (m *ownerManager) watchOwner(ctx context.Context, etcdSession *concurrency.Session, key string) {
logPrefix := fmt.Sprintf("[%s] ownerManager %s watch owner key %v", m.prompt, m.id, key)
logCtx := logutil.WithKeyValue(context.Background(), "owner info", logPrefix)
logutil.BgLogger().Debug(logPrefix)
watchCh := m.etcdCli.Watch(ctx, key)
for {
select {
case resp, ok := <-watchCh:
if !ok {
metrics.WatchOwnerCounter.WithLabelValues(m.prompt, metrics.WatcherClosed).Inc()
logutil.Logger(logCtx).Info("watcher is closed, no owner")
return
}
if resp.Canceled {
metrics.WatchOwnerCounter.WithLabelValues(m.prompt, metrics.Cancelled).Inc()
logutil.Logger(logCtx).Info("watch canceled, no owner")
return
}
for _, ev := range resp.Events {
if ev.Type == mvccpb.DELETE {
metrics.WatchOwnerCounter.WithLabelValues(m.prompt, metrics.Deleted).Inc()
logutil.Logger(logCtx).Info("watch failed, owner is deleted")
return
}
}
case <-etcdSession.Done():
metrics.WatchOwnerCounter.WithLabelValues(m.prompt, metrics.SessionDone).Inc()
return
case <-ctx.Done():
metrics.WatchOwnerCounter.WithLabelValues(m.prompt, metrics.CtxDone).Inc()
return
}
}
}
func init() {
err := setManagerSessionTTL()
if err != nil {
logutil.BgLogger().Warn("set manager session TTL failed", zap.Error(err))
}
}
func contextDone(ctx context.Context, err error) error {
select {
case <-ctx.Done():
return errors.Trace(ctx.Err())
default:
}
// Sometime the ctx isn't closed, but the etcd client is closed,
// we need to treat it as if context is done.
// TODO: Make sure ctx is closed with etcd client.
if terror.ErrorEqual(err, context.Canceled) ||
terror.ErrorEqual(err, context.DeadlineExceeded) ||
terror.ErrorEqual(err, grpc.ErrClientConnClosing) {
return errors.Trace(err)
}
return nil
}