You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
110 lines
2.3 KiB
110 lines
2.3 KiB
// Copyright 2016 PingCAP, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/pingcap/log"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
func addJobs(jobCount int, jobChan chan struct{}) {
|
|
for i := 0; i < jobCount; i++ {
|
|
jobChan <- struct{}{}
|
|
}
|
|
|
|
close(jobChan)
|
|
}
|
|
|
|
func doInsert(table *table, db *sql.DB, count int) {
|
|
sqls, err := genRowDatas(table, count)
|
|
if err != nil {
|
|
log.Fatal("generate data failed", zap.Error(err))
|
|
}
|
|
|
|
txn, err := db.Begin()
|
|
if err != nil {
|
|
log.Fatal("begin failed", zap.Error(err))
|
|
}
|
|
|
|
for _, sql := range sqls {
|
|
_, err = txn.Exec(sql)
|
|
if err != nil {
|
|
log.Fatal("exec failed", zap.Error(err))
|
|
}
|
|
}
|
|
|
|
err = txn.Commit()
|
|
if err != nil {
|
|
log.Fatal("commit failed", zap.Error(err))
|
|
}
|
|
}
|
|
|
|
func doJob(table *table, db *sql.DB, batch int, jobChan chan struct{}, doneChan chan struct{}) {
|
|
count := 0
|
|
for range jobChan {
|
|
count++
|
|
if count == batch {
|
|
doInsert(table, db, count)
|
|
count = 0
|
|
}
|
|
}
|
|
|
|
if count > 0 {
|
|
doInsert(table, db, count)
|
|
}
|
|
|
|
doneChan <- struct{}{}
|
|
}
|
|
|
|
func doWait(doneChan chan struct{}, start time.Time, jobCount int, workerCount int) {
|
|
for i := 0; i < workerCount; i++ {
|
|
<-doneChan
|
|
}
|
|
|
|
close(doneChan)
|
|
|
|
now := time.Now()
|
|
seconds := now.Unix() - start.Unix()
|
|
|
|
tps := int64(-1)
|
|
if seconds > 0 {
|
|
tps = int64(jobCount) / seconds
|
|
}
|
|
|
|
fmt.Printf("[importer]total %d cases, cost %d seconds, tps %d, start %s, now %s\n", jobCount, seconds, tps, start, now)
|
|
}
|
|
|
|
func doProcess(table *table, dbs []*sql.DB, jobCount int, workerCount int, batch int) {
|
|
jobChan := make(chan struct{}, 16*workerCount)
|
|
doneChan := make(chan struct{}, workerCount)
|
|
|
|
start := time.Now()
|
|
go addJobs(jobCount, jobChan)
|
|
|
|
for _, col := range table.columns {
|
|
if col.incremental {
|
|
workerCount = 1
|
|
break
|
|
}
|
|
}
|
|
for i := 0; i < workerCount; i++ {
|
|
go doJob(table, dbs[i], batch, jobChan, doneChan)
|
|
}
|
|
|
|
doWait(doneChan, start, jobCount, workerCount)
|
|
}
|
|
|