You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
88 lines
2.7 KiB
88 lines
2.7 KiB
package service
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"github.com/IBM/sarama"
|
|
"github.com/TremblingV5/DouTok/applications/message/pack"
|
|
"github.com/TremblingV5/DouTok/pkg/misc"
|
|
"github.com/TremblingV5/DouTok/pkg/utils"
|
|
"github.com/cloudwego/kitex/pkg/klog"
|
|
)
|
|
|
|
type msgConsumerGroup struct{}
|
|
|
|
func (m msgConsumerGroup) Setup(_ sarama.ConsumerGroupSession) error { return nil }
|
|
func (m msgConsumerGroup) Cleanup(_ sarama.ConsumerGroupSession) error { return nil }
|
|
func (m msgConsumerGroup) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
|
|
for msg := range claim.Messages() {
|
|
fmt.Printf("Message topic:%q partition:%d offset:%d value:%s\n", msg.Topic, msg.Partition, msg.Offset, string(msg.Value))
|
|
|
|
message := pack.Message{}
|
|
json.Unmarshal(msg.Value, &message)
|
|
mp, err := misc.Struct2Map(message)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Struct2Map 有bug,会转float64
|
|
klog.Infof("messages to map, msg content = %s from %d to %d\n", mp["content"], int64(mp["from_user_id"].(float64)), int64(mp["to_user_id"].(float64)))
|
|
sessionId := utils.GenerateSessionId(message.FromUserId, message.ToUserId)
|
|
|
|
// 更新 redis 的最新消息
|
|
err = RedisClient.HMSet(context.Background(), sessionId, mp).Err()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
content, err := RedisClient.HGet(context.Background(), sessionId, "content").Result()
|
|
if err != nil {
|
|
klog.Errorf("get friend list message error, err = %s", err)
|
|
}
|
|
klog.Infof("content = %s\n", content)
|
|
|
|
// 将消息存入 hbase
|
|
// 生成 rowkey
|
|
rowKey := fmt.Sprintf("%s%d", sessionId, message.CreateTime)
|
|
|
|
println("consume msg form kafka, generate rowKey = ", rowKey)
|
|
|
|
// 构造 hbase 一条数据
|
|
hbData := map[string]map[string][]byte{
|
|
"data": {
|
|
"id": []byte(fmt.Sprintf("%d", message.Id)),
|
|
"from_user_id": []byte(fmt.Sprintf("%d", message.FromUserId)),
|
|
"to_user_id": []byte(fmt.Sprintf("%d", message.ToUserId)),
|
|
"content": []byte(message.Content),
|
|
"create_time": []byte(fmt.Sprintf("%d", message.CreateTime)),
|
|
},
|
|
}
|
|
|
|
err = HBClient.Put(ViperConfig.Viper.GetString("Hbase.Table"), rowKey, hbData)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// TODO 1、解决消息重复消费 2、分布式事务一致性(redis 与 hbase 同时成功或失败、重试策略)
|
|
|
|
// 标记,sarama会自动进行提交,默认间隔1秒
|
|
sess.MarkMessage(msg, "")
|
|
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var consumerGroup msgConsumerGroup
|
|
|
|
func ConsumeMsg() {
|
|
|
|
for {
|
|
err := ConsumerGroup.Consume(context.Background(), ViperConfig.Viper.GetStringSlice("Kafka.Topics"), consumerGroup)
|
|
if err != nil {
|
|
fmt.Println(err.Error())
|
|
break
|
|
}
|
|
}
|
|
|
|
_ = ConsumerGroup.Close()
|
|
}
|
|
|