merge v3dev into main (#504)

* statistics user register * refactor: router change * minio init * UserRegisterCount * push use local conn * refactor: user pb update * remove online push close grpc conn * refactor: user pb update * refactor:pb file * msgs statistics * msgs statistics * revoke userID * refactor: errcode update * active user * active user * active user * refactor: errcode update * feat: conn update token * active user * active user * feat: conn update token * active user * feat: conn update token * feat: conn update token * feat: conn update token * add tx_oss cos * active user * active user * group create * group create * feat: group notification show to conversation * feat: group notification show to conversation * group active * user active * sendNotificationWithName * withname * privateChat * a2r call option * grpc with detail return error * change log error * chain unary interceptor * api nil slice map * fix sync has read * fix: text update * fix: update add model * set conversations update * set privateChat * fix: content update * remove unuse rpc * msgDestruct * cron use rpc mw * set IsMsgDestruct * msg destruct * msgDestruct * s3 minio, cos, oss support * feat: add implement of GetUsersOnlineStatus, #472 (#477) * s3 minio, cos, oss support * s3 route * remove extendMsg code * s3 route * remove unuse code * s3 pb * s3 pb * s3 pb * s3 presigned put * s3 presigned test * s3 presigned test * s3 presigned test * s3 presigned test * s3 presigned test * s3 presigned test * s3 presigned test * s3 presigned test * Update .gitignore (#482) * s3 debug log * s3 debug log * cron add log and fix cron * add log * cron * s3 config * fix kick user bug * s3 cos * add kick log * s3 cos test * s3 cos test * s3 cos test * kick user log * kickuserlog * s3 cos copy * s3 cos copy * s3 url * s3 url * s3 AccessURL * log * s3 InitiateMultipartUpload add ExpireTime --------- Co-authored-by: withchao <993506633@qq.com> Co-authored-by: wangchuxiao <wangchuxiao97@outlook.com> Co-authored-by: BanTanger <88583317+BanTanger@users.noreply.github.com> Co-authored-by: withchao <48119764+withchao@users.noreply.github.com> Co-authored-by: Alan <68671759+hanzhixiao@users.noreply.github.com>
2026-04-28 06:19:20 +08:00 · 2023-07-12 15:31:24 +08:00
parent 5467887e1c
commit 454bf97e3f
123 changed files with 7377 additions and 10288 deletions
@@ -1,21 +1,6 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
 package controller

 import (
-	"fmt"
 	"time"

 	"github.com/redis/go-redis/v9"
@@ -33,11 +18,10 @@ import (
 	"context"
 	"errors"

-	"go.mongodb.org/mongo-driver/mongo"
-
 	pbMsg "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
+	"go.mongodb.org/mongo-driver/mongo"
 )

 const (
@@ -56,28 +40,17 @@ type CommonMsgDatabase interface {
 	DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error
 	DelUserDeleteMsgsList(ctx context.Context, conversationID string, seqs []int64)
 	// incrSeq然后批量插入缓存
-	BatchInsertChat2Cache(
-		ctx context.Context,
-		conversationID string,
-		msgs []*sdkws.MsgData,
-	) (seq int64, isNewConversation bool, err error)
+	BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, err error)

 	//  通过seqList获取mongo中写扩散消息
-	GetMsgBySeqsRange(
-		ctx context.Context,
-		userID string,
-		conversationID string,
-		begin, end, num, userMaxSeq int64,
-	) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error)
+	GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error)
 	// 通过seqList获取大群在 mongo里面的消息
-	GetMsgBySeqs(
-		ctx context.Context,
-		userID string,
-		conversationID string,
-		seqs []int64,
-	) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error)
+	GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error)
 	// 删除会话消息重置最小seq， remainTime为消息保留的时间单位秒,超时消息删除， 传0删除所有消息(此方法不删除redis cache)
 	DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error
+	// 用户标记删除过期消息返回标记删除的seq列表
+	UserMsgsDestruct(cte context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error)
+
 	// 用户根据seq删除消息
 	DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error
 	// 物理删除消息置空
@@ -101,11 +74,8 @@ type CommonMsgDatabase interface {
 	GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error)
 	UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error

-	GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (maxSeq, minSeq int64, err error)
-	GetConversationMinMaxSeqInMongoAndCache(
-		ctx context.Context,
-		conversationID string,
-	) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error)
+	GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error)
+	GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error)
 	SetSendMsgStatus(ctx context.Context, id string, status int32) error
 	GetSendMsgStatus(ctx context.Context, id string) (int32, error)

@@ -115,51 +85,17 @@ type CommonMsgDatabase interface {
 	MsgToPushMQ(ctx context.Context, key, conversarionID string, msg2mq *sdkws.MsgData) (int32, int64, error)
 	MsgToMongoMQ(ctx context.Context, key, conversarionID string, msgs []*sdkws.MsgData, lastSeq int64) error

-	// modify
-	JudgeMessageReactionExist(ctx context.Context, clientMsgID string, sessionType int32) (bool, error)
-	SetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey, value string) error
-	SetMessageReactionExpire(
-		ctx context.Context,
-		clientMsgID string,
-		sessionType int32,
-		expiration time.Duration,
-	) (bool, error)
-	GetExtendMsg(
-		ctx context.Context,
-		conversationID string,
-		sessionType int32,
-		clientMsgID string,
-		maxMsgUpdateTime int64,
-	) (*pbMsg.ExtendMsg, error)
-	InsertOrUpdateReactionExtendMsgSet(
-		ctx context.Context,
-		conversationID string,
-		sessionType int32,
-		clientMsgID string,
-		msgFirstModifyTime int64,
-		reactionExtensionList map[string]*sdkws.KeyValue,
-	) error
-	GetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey string) (string, error)
-	GetOneMessageAllReactionList(ctx context.Context, clientMsgID string, sessionType int32) (map[string]string, error)
-	DeleteOneMessageKey(ctx context.Context, clientMsgID string, sessionType int32, subKey string) error
-	DeleteReactionExtendMsgSet(
-		ctx context.Context,
-		conversationID string,
-		sessionType int32,
-		clientMsgID string,
-		msgFirstModifyTime int64,
-		reactionExtensionList map[string]*sdkws.KeyValue,
-	) error
+	RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*unRelationTb.UserCount, dateCount map[string]int64, err error)
+	RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*unRelationTb.GroupCount, dateCount map[string]int64, err error)
 }

 func NewCommonMsgDatabase(msgDocModel unRelationTb.MsgDocModelInterface, cacheModel cache.MsgModel) CommonMsgDatabase {
 	return &commonMsgDatabase{
-		msgDocDatabase:   msgDocModel,
-		cache:            cacheModel,
-		producer:         kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.LatestMsgToRedis.Topic),
-		producerToMongo:  kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToMongo.Topic),
-		producerToPush:   kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToPush.Topic),
-		producerToModify: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToModify.Topic),
+		msgDocDatabase:  msgDocModel,
+		cache:           cacheModel,
+		producer:        kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.LatestMsgToRedis.Topic),
+		producerToMongo: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToMongo.Topic),
+		producerToPush:  kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToPush.Topic),
 	}
 }

@@ -171,15 +107,13 @@ func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database)
 }

 type commonMsgDatabase struct {
-	msgDocDatabase    unRelationTb.MsgDocModelInterface
-	extendMsgDatabase unRelationTb.ExtendMsgSetModelInterface
-	extendMsgSetModel unRelationTb.ExtendMsgSetModel
-	msg               unRelationTb.MsgDocModel
-	cache             cache.MsgModel
-	producer          *kafka.Producer
-	producerToMongo   *kafka.Producer
-	producerToModify  *kafka.Producer
-	producerToPush    *kafka.Producer
+	msgDocDatabase   unRelationTb.MsgDocModelInterface
+	msg              unRelationTb.MsgDocModel
+	cache            cache.MsgModel
+	producer         *kafka.Producer
+	producerToMongo  *kafka.Producer
+	producerToModify *kafka.Producer
+	producerToPush   *kafka.Producer
 }

 func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error {
@@ -187,32 +121,16 @@ func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sd
 	return err
 }

-func (db *commonMsgDatabase) MsgToModifyMQ(
-	ctx context.Context,
-	key, conversationID string,
-	messages []*sdkws.MsgData,
-) error {
+func (db *commonMsgDatabase) MsgToModifyMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData) error {
 	if len(messages) > 0 {
-		_, _, err := db.producerToModify.SendMessage(
-			ctx,
-			key,
-			&pbMsg.MsgDataToModifyByMQ{ConversationID: conversationID, Messages: messages},
-		)
+		_, _, err := db.producerToModify.SendMessage(ctx, key, &pbMsg.MsgDataToModifyByMQ{ConversationID: conversationID, Messages: messages})
 		return err
 	}
 	return nil
 }

-func (db *commonMsgDatabase) MsgToPushMQ(
-	ctx context.Context,
-	key, conversationID string,
-	msg2mq *sdkws.MsgData,
-) (int32, int64, error) {
-	partition, offset, err := db.producerToPush.SendMessage(
-		ctx,
-		key,
-		&pbMsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID},
-	)
+func (db *commonMsgDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) {
+	partition, offset, err := db.producerToPush.SendMessage(ctx, key, &pbMsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID})
 	if err != nil {
 		log.ZError(ctx, "MsgToPushMQ", err, "key", key, "msg2mq", msg2mq)
 		return 0, 0, err
@@ -220,30 +138,15 @@ func (db *commonMsgDatabase) MsgToPushMQ(
 	return partition, offset, nil
 }

-func (db *commonMsgDatabase) MsgToMongoMQ(
-	ctx context.Context,
-	key, conversationID string,
-	messages []*sdkws.MsgData,
-	lastSeq int64,
-) error {
+func (db *commonMsgDatabase) MsgToMongoMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData, lastSeq int64) error {
 	if len(messages) > 0 {
-		_, _, err := db.producerToMongo.SendMessage(
-			ctx,
-			key,
-			&pbMsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages},
-		)
+		_, _, err := db.producerToMongo.SendMessage(ctx, key, &pbMsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages})
 		return err
 	}
 	return nil
 }

-func (db *commonMsgDatabase) BatchInsertBlock(
-	ctx context.Context,
-	conversationID string,
-	fields []any,
-	key int8,
-	firstSeq int64,
-) error {
+func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error {
 	if len(fields) == 0 {
 		return nil
 	}
@@ -344,12 +247,7 @@ func (db *commonMsgDatabase) BatchInsertBlock(
 	return nil
 }

-func (db *commonMsgDatabase) BatchInsertChat2DB(
-	ctx context.Context,
-	conversationID string,
-	msgList []*sdkws.MsgData,
-	currentMaxSeq int64,
-) error {
+func (db *commonMsgDatabase) BatchInsertChat2DB(ctx context.Context, conversationID string, msgList []*sdkws.MsgData, currentMaxSeq int64) error {
 	if len(msgList) == 0 {
 		return errs.ErrArgs.Wrap("msgList is empty")
 	}
@@ -395,21 +293,11 @@ func (db *commonMsgDatabase) BatchInsertChat2DB(
 	return db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq)
 }

-func (db *commonMsgDatabase) RevokeMsg(
-	ctx context.Context,
-	conversationID string,
-	seq int64,
-	revoke *unRelationTb.RevokeModel,
-) error {
+func (db *commonMsgDatabase) RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *unRelationTb.RevokeModel) error {
 	return db.BatchInsertBlock(ctx, conversationID, []any{revoke}, updateKeyRevoke, seq)
 }

-func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(
-	ctx context.Context,
-	userID string,
-	conversationID string,
-	totalSeqs []int64,
-) error {
+func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, totalSeqs []int64) error {
 	for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, totalSeqs) {
 		var indexes []int64
 		for _, seq := range seqs {
@@ -432,11 +320,7 @@ func (db *commonMsgDatabase) DelUserDeleteMsgsList(ctx context.Context, conversa
 	db.cache.DelUserDeleteMsgsList(ctx, conversationID, seqs)
 }

-func (db *commonMsgDatabase) BatchInsertChat2Cache(
-	ctx context.Context,
-	conversationID string,
-	msgs []*sdkws.MsgData,
-) (seq int64, isNew bool, err error) {
+func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) {
 	currentMaxSeq, err := db.cache.GetMaxSeq(ctx, conversationID)
 	if err != nil && errs.Unwrap(err) != redis.Nil {
 		prome.Inc(prome.SeqGetFailedCounter)
@@ -483,11 +367,7 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(
 	return lastMaxSeq, isNew, utils.Wrap(err, "")
 }

-func (db *commonMsgDatabase) getMsgBySeqs(
-	ctx context.Context,
-	userID, conversationID string,
-	seqs []int64,
-) (totalMsgs []*sdkws.MsgData, err error) {
+func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversationID string, seqs []int64) (totalMsgs []*sdkws.MsgData, err error) {
 	for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, seqs) {
 		//log.ZDebug(ctx, "getMsgBySeqs", "docID", docID, "seqs", seqs)
 		msgs, err := db.findMsgInfoBySeq(ctx, userID, docID, seqs)
@@ -501,50 +381,7 @@ func (db *commonMsgDatabase) getMsgBySeqs(
 	return totalMsgs, nil
 }

-// func (db *commonMsgDatabase) refetchDelSeqsMsgs(ctx context.Context, conversationID string, delNums, rangeBegin,
-// begin int64) (seqMsgs []*unRelationTb.MsgDataModel, err error) {
-// 	var reFetchSeqs []int64
-// 	if delNums > 0 {
-// 		newBeginSeq := rangeBegin - delNums
-// 		if newBeginSeq >= begin {
-// 			newEndSeq := rangeBegin - 1
-// 			for i := newBeginSeq; i <= newEndSeq; i++ {
-// 				reFetchSeqs = append(reFetchSeqs, i)
-// 			}
-// 		}
-// 	}
-// 	if len(reFetchSeqs) == 0 {
-// 		return
-// 	}
-// 	if len(reFetchSeqs) > 0 {
-// 		m := db.msg.GetDocIDSeqsMap(conversationID, reFetchSeqs)
-// 		for docID, seqs := range m {
-// 			msgs, _, err := db.findMsgInfoBySeq(ctx, docID, seqs)
-// 			if err != nil {
-// 				return nil, err
-// 			}
-// 			for _, msg := range msgs {
-// 				if msg.Status != constant.MsgDeleted {
-// 					seqMsgs = append(seqMsgs, msg)
-// 				}
-// 			}
-// 		}
-// 	}
-// 	if len(seqMsgs) < int(delNums) {
-// 		seqMsgs2, err := db.refetchDelSeqsMsgs(ctx, conversationID, delNums-int64(len(seqMsgs)), rangeBegin-1, begin)
-// 		if err != nil {
-// 			return seqMsgs, err
-// 		}
-// 		seqMsgs = append(seqMsgs, seqMsgs2...)
-// 	}
-// 	return seqMsgs, nil
-// }
-
-func (db *commonMsgDatabase) findMsgInfoBySeq(
-	ctx context.Context,
-	userID, docID string,
-	seqs []int64,
-) (totalMsgs []*unRelationTb.MsgInfoModel, err error) {
+func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, seqs []int64) (totalMsgs []*unRelationTb.MsgInfoModel, err error) {
 	msgs, err := db.msgDocDatabase.GetMsgBySeqIndexIn1Doc(ctx, userID, docID, seqs)
 	for _, msg := range msgs {
 		if msg.IsRead {
@@ -554,25 +391,8 @@ func (db *commonMsgDatabase) findMsgInfoBySeq(
 	return msgs, err
 }

-func (db *commonMsgDatabase) getMsgBySeqsRange(
-	ctx context.Context,
-	userID string,
-	conversationID string,
-	allSeqs []int64,
-	begin, end int64,
-) (seqMsgs []*sdkws.MsgData, err error) {
-	log.ZDebug(
-		ctx,
-		"getMsgBySeqsRange",
-		"conversationID",
-		conversationID,
-		"allSeqs",
-		allSeqs,
-		"begin",
-		begin,
-		"end",
-		end,
-	)
+func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID string, conversationID string, allSeqs []int64, begin, end int64) (seqMsgs []*sdkws.MsgData, err error) {
+	log.ZDebug(ctx, "getMsgBySeqsRange", "conversationID", conversationID, "allSeqs", allSeqs, "begin", begin, "end", end)
 	for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, allSeqs) {
 		log.ZDebug(ctx, "getMsgBySeqsRange", "docID", docID, "seqs", seqs)
 		msgs, err := db.findMsgInfoBySeq(ctx, userID, docID, seqs)
@@ -589,12 +409,7 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(
 	return seqMsgs, nil
 }

-func (db *commonMsgDatabase) GetMsgBySeqsRange(
-	ctx context.Context,
-	userID string,
-	conversationID string,
-	begin, end, num, userMaxSeq int64,
-) (int64, int64, []*sdkws.MsgData, error) {
+func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) {
 	userMinSeq, err := db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
 	if err != nil && errs.Unwrap(err) != redis.Nil {
 		return 0, 0, nil, err
@@ -614,18 +429,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(
 	if err != nil && errs.Unwrap(err) != redis.Nil {
 		return 0, 0, nil, err
 	}
-	log.ZDebug(
-		ctx,
-		"GetMsgBySeqsRange",
-		"userMinSeq",
-		userMinSeq,
-		"conMinSeq",
-		minSeq,
-		"conMaxSeq",
-		maxSeq,
-		"userMaxSeq",
-		userMaxSeq,
-	)
+	log.ZDebug(ctx, "GetMsgBySeqsRange", "userMinSeq", userMinSeq, "conMinSeq", minSeq, "conMaxSeq", maxSeq, "userMaxSeq", userMaxSeq)
 	if userMaxSeq != 0 {
 		if userMaxSeq < maxSeq {
 			maxSeq = userMaxSeq
@@ -675,18 +479,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(
 				cacheDelNum += 1
 			}
 		}
-		log.ZDebug(
-			ctx,
-			"get delSeqs from redis",
-			"delSeqs",
-			delSeqs,
-			"userID",
-			userID,
-			"conversationID",
-			conversationID,
-			"cacheDelNum",
-			cacheDelNum,
-		)
+		log.ZDebug(ctx, "get delSeqs from redis", "delSeqs", delSeqs, "userID", userID, "conversationID", conversationID, "cacheDelNum", cacheDelNum)
 		var reGetSeqsCache []int64
 		for i := 1; i <= cacheDelNum; {
 			newSeq := newBegin - int64(i)
@@ -706,15 +499,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(
 			if err != nil {
 				if err != redis.Nil {
 					prome.Add(prome.MsgPullFromRedisFailedCounter, len(failedSeqs2))
-					log.ZError(
-						ctx,
-						"get message from redis exception",
-						err,
-						"conversationID",
-						conversationID,
-						"seqs",
-						reGetSeqsCache,
-					)
+					log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", reGetSeqsCache)
 				}
 			}
 			failedSeqs = append(failedSeqs, failedSeqs2...)
@@ -740,12 +525,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(
 	return minSeq, maxSeq, successMsgs, nil
 }

-func (db *commonMsgDatabase) GetMsgBySeqs(
-	ctx context.Context,
-	userID string,
-	conversationID string,
-	seqs []int64,
-) (int64, int64, []*sdkws.MsgData, error) {
+func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (int64, int64, []*sdkws.MsgData, error) {
 	userMinSeq, err := db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
 	if err != nil && errs.Unwrap(err) != redis.Nil {
 		return 0, 0, nil, err
@@ -771,33 +551,10 @@ func (db *commonMsgDatabase) GetMsgBySeqs(
 	if err != nil {
 		if err != redis.Nil {
 			prome.Add(prome.MsgPullFromRedisFailedCounter, len(failedSeqs))
-			log.ZError(
-				ctx,
-				"get message from redis exception",
-				err,
-				"failedSeqs",
-				failedSeqs,
-				"conversationID",
-				conversationID,
-			)
+			log.ZError(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID)
 		}
 	}
-	log.ZInfo(
-		ctx,
-		"db.cache.GetMessagesBySeq",
-		"userID",
-		userID,
-		"conversationID",
-		conversationID,
-		"seqs",
-		seqs,
-		"successMsgs",
-		len(successMsgs),
-		"failedSeqs",
-		failedSeqs,
-		"conversationID",
-		conversationID,
-	)
+	log.ZInfo(ctx, "db.cache.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs", seqs, "successMsgs", len(successMsgs), "failedSeqs", failedSeqs, "conversationID", conversationID)
 	prome.Add(prome.MsgPullFromRedisSuccessCounter, len(successMsgs))
 	if len(failedSeqs) > 0 {
 		mongoMsgs, err := db.getMsgBySeqs(ctx, userID, conversationID, failedSeqs)
@@ -811,11 +568,7 @@ func (db *commonMsgDatabase) GetMsgBySeqs(
 	return minSeq, maxSeq, successMsgs, nil
 }

-func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(
-	ctx context.Context,
-	conversationID string,
-	remainTime int64,
-) error {
+func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error {
 	var delStruct delMsgRecursionStruct
 	var skip int64
 	minSeq, err := db.deleteMsgRecursion(ctx, conversationID, skip, &delStruct, remainTime)
@@ -835,6 +588,49 @@ func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(
 	return db.cache.SetMinSeq(ctx, conversationID, minSeq)
 }

+func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error) {
+	var index int64
+	for {
+		// from oldest 2 newest
+		msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1)
+		if err != nil || msgDocModel.DocID == "" {
+			if err != nil {
+				if err == unrelation.ErrMsgListNotExist {
+					log.ZDebug(ctx, "deleteMsgRecursion finished", "conversationID", conversationID, "userID", userID, "index", index)
+				} else {
+					log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index)
+				}
+			}
+			// 获取报错，或者获取不到了，物理删除并且返回seq delMongoMsgsPhysical(delStruct.delDocIDList), 结束递归
+			break
+		}
+		index++
+		//&& msgDocModel.Msg[0].Msg.SendTime > lastMsgDestructTime.UnixMilli()
+		if len(msgDocModel.Msg) > 0 {
+			for _, msg := range msgDocModel.Msg {
+				if msg != nil && msg.Msg != nil && msg.Msg.SendTime+destructTime*1000 <= time.Now().UnixMilli() {
+					if msg.Msg.SendTime > lastMsgDestructTime.UnixMilli() && !utils.Contain(userID, msg.DelList...) {
+						seqs = append(seqs, msg.Msg.Seq)
+					}
+				} else {
+					log.ZDebug(ctx, "deleteMsgRecursion finished", "conversationID", conversationID, "userID", userID, "index", index)
+					break
+				}
+
+			}
+		}
+	}
+
+	log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversationID, "userID", userID, "seqs", seqs)
+	if len(seqs) > 0 {
+		latestSeq := seqs[len(seqs)-1]
+		if err := db.cache.SetConversationUserMinSeq(ctx, conversationID, userID, latestSeq); err != nil {
+			return nil, err
+		}
+	}
+	return seqs, nil
+}
+
 // this is struct for recursion
 type delMsgRecursionStruct struct {
 	minSeq    int64
@@ -849,26 +645,13 @@ func (d *delMsgRecursionStruct) getSetMinSeq() int64 {
 // seq 70
 // set minSeq 21
 // recursion 删除list并且返回设置的最小seq
-func (db *commonMsgDatabase) deleteMsgRecursion(
-	ctx context.Context,
-	conversationID string,
-	index int64,
-	delStruct *delMsgRecursionStruct,
-	remainTime int64,
-) (int64, error) {
+func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversationID string, index int64, delStruct *delMsgRecursionStruct, remainTime int64) (int64, error) {
 	// find from oldest list
 	msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1)
 	if err != nil || msgDocModel.DocID == "" {
 		if err != nil {
 			if err == unrelation.ErrMsgListNotExist {
-				log.ZDebug(
-					ctx,
-					"deleteMsgRecursion ErrMsgListNotExist",
-					"conversationID",
-					conversationID,
-					"index:",
-					index,
-				)
+				log.ZDebug(ctx, "deleteMsgRecursion ErrMsgListNotExist", "conversationID", conversationID, "index:", index)
 			} else {
 				log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index)
 			}
@@ -880,23 +663,11 @@ func (db *commonMsgDatabase) deleteMsgRecursion(
 		}
 		return delStruct.getSetMinSeq() + 1, nil
 	}
-	log.ZDebug(
-		ctx,
-		"doc info",
-		"conversationID",
-		conversationID,
-		"index",
-		index,
-		"docID",
-		msgDocModel.DocID,
-		"len",
-		len(msgDocModel.Msg),
-	)
+	log.ZDebug(ctx, "doc info", "conversationID", conversationID, "index", index, "docID", msgDocModel.DocID, "len", len(msgDocModel.Msg))
 	if int64(len(msgDocModel.Msg)) > db.msg.GetSingleGocMsgNum() {
 		log.ZWarn(ctx, "msgs too large", nil, "lenth", len(msgDocModel.Msg), "docID:", msgDocModel.DocID)
 	}
-	if msgDocModel.IsFull() &&
-		msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < utils.GetCurrentTimestampByMill() {
+	if msgDocModel.IsFull() && msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < utils.GetCurrentTimestampByMill() {
 		log.ZDebug(ctx, "doc is full and all msg is expired", "docID", msgDocModel.DocID)
 		delStruct.delDocIDs = append(delStruct.delDocIDs, msgDocModel.DocID)
 		delStruct.minSeq = msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.Seq
@@ -933,11 +704,7 @@ func (db *commonMsgDatabase) deleteMsgRecursion(
 	return seq, err
 }

-func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(
-	ctx context.Context,
-	conversationID string,
-	allSeqs []int64,
-) error {
+func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, allSeqs []int64) error {
 	if err := db.cache.DeleteMessages(ctx, conversationID, allSeqs); err != nil {
 		return err
 	}
@@ -953,12 +720,7 @@ func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(
 	return nil
 }

-func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(
-	ctx context.Context,
-	userID string,
-	conversationID string,
-	seqs []int64,
-) error {
+func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error {
 	cachedMsgs, _, err := db.cache.GetMessagesBySeq(ctx, conversationID, seqs)
 	if err != nil && errs.Unwrap(err) != redis.Nil {
 		log.ZWarn(ctx, "DeleteUserMsgsBySeqs", err, "conversationID", conversationID, "seqs", seqs)
@@ -1027,70 +789,31 @@ func (db *commonMsgDatabase) GetMinSeqs(ctx context.Context, conversationIDs []s
 func (db *commonMsgDatabase) GetMinSeq(ctx context.Context, conversationID string) (int64, error) {
 	return db.cache.GetMinSeq(ctx, conversationID)
 }
-
-func (db *commonMsgDatabase) GetConversationUserMinSeq(
-	ctx context.Context,
-	conversationID string,
-	userID string,
-) (int64, error) {
+func (db *commonMsgDatabase) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
 	return db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
 }
-
-func (db *commonMsgDatabase) GetConversationUserMinSeqs(
-	ctx context.Context,
-	conversationID string,
-	userIDs []string,
-) (map[string]int64, error) {
+func (db *commonMsgDatabase) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) {
 	return db.cache.GetConversationUserMinSeqs(ctx, conversationID, userIDs)
 }
-
-func (db *commonMsgDatabase) SetConversationUserMinSeq(
-	ctx context.Context,
-	conversationID string,
-	userID string,
-	minSeq int64,
-) error {
+func (db *commonMsgDatabase) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error {
 	return db.cache.SetConversationUserMinSeq(ctx, conversationID, userID, minSeq)
 }
-
-func (db *commonMsgDatabase) SetConversationUserMinSeqs(
-	ctx context.Context,
-	conversationID string,
-	seqs map[string]int64,
-) (err error) {
+func (db *commonMsgDatabase) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) {
 	return db.cache.SetConversationUserMinSeqs(ctx, conversationID, seqs)
 }

-func (db *commonMsgDatabase) SetUserConversationsMinSeqs(
-	ctx context.Context,
-	userID string,
-	seqs map[string]int64,
-) error {
+func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error {
 	return db.cache.SetUserConversationsMinSeqs(ctx, userID, seqs)
 }

-func (db *commonMsgDatabase) UserSetHasReadSeqs(
-	ctx context.Context,
-	userID string,
-	hasReadSeqs map[string]int64,
-) error {
+func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error {
 	return db.cache.UserSetHasReadSeqs(ctx, userID, hasReadSeqs)
 }

-func (db *commonMsgDatabase) SetHasReadSeq(
-	ctx context.Context,
-	userID string,
-	conversationID string,
-	hasReadSeq int64,
-) error {
+func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error {
 	return db.cache.SetHasReadSeq(ctx, userID, conversationID, hasReadSeq)
 }
-
-func (db *commonMsgDatabase) GetHasReadSeqs(
-	ctx context.Context,
-	userID string,
-	conversationIDs []string,
-) (map[string]int64, error) {
+func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) {
 	return db.cache.GetHasReadSeqs(ctx, userID, conversationIDs)
 }

@@ -1106,10 +829,7 @@ func (db *commonMsgDatabase) GetSendMsgStatus(ctx context.Context, id string) (i
 	return db.cache.GetSendMsgStatus(ctx, id)
 }

-func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(
-	ctx context.Context,
-	conversationID string,
-) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) {
+func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) {
 	minSeqMongo, maxSeqMongo, err = db.GetMinMaxSeqMongo(ctx, conversationID)
 	if err != nil {
 		return
@@ -1125,17 +845,11 @@ func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(
 	return
 }

-func (db *commonMsgDatabase) GetMongoMaxAndMinSeq(
-	ctx context.Context,
-	conversationID string,
-) (maxSeq, minSeq int64, err error) {
+func (db *commonMsgDatabase) GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) {
 	return db.GetMinMaxSeqMongo(ctx, conversationID)
 }

-func (db *commonMsgDatabase) GetMinMaxSeqMongo(
-	ctx context.Context,
-	conversationID string,
-) (minSeqMongo, maxSeqMongo int64, err error) {
+func (db *commonMsgDatabase) GetMinMaxSeqMongo(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) {
 	oldestMsgMongo, err := db.msgDocDatabase.GetOldestMsg(ctx, conversationID)
 	if err != nil {
 		return
@@ -1149,124 +863,10 @@ func (db *commonMsgDatabase) GetMinMaxSeqMongo(
 	return
 }

-func (db *commonMsgDatabase) JudgeMessageReactionExist(
-	ctx context.Context,
-	clientMsgID string,
-	sessionType int32,
-) (bool, error) {
-	return db.cache.JudgeMessageReactionExist(ctx, clientMsgID, sessionType)
+func (db *commonMsgDatabase) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*unRelationTb.UserCount, dateCount map[string]int64, err error) {
+	return db.msgDocDatabase.RangeUserSendCount(ctx, start, end, group, ase, pageNumber, showNumber)
 }

-func (db *commonMsgDatabase) SetMessageTypeKeyValue(
-	ctx context.Context,
-	clientMsgID string,
-	sessionType int32,
-	typeKey, value string,
-) error {
-	return db.cache.SetMessageTypeKeyValue(ctx, clientMsgID, sessionType, typeKey, value)
-}
-
-func (db *commonMsgDatabase) SetMessageReactionExpire(
-	ctx context.Context,
-	clientMsgID string,
-	sessionType int32,
-	expiration time.Duration,
-) (bool, error) {
-	return db.cache.SetMessageReactionExpire(ctx, clientMsgID, sessionType, expiration)
-}
-
-func (db *commonMsgDatabase) GetMessageTypeKeyValue(
-	ctx context.Context,
-	clientMsgID string,
-	sessionType int32,
-	typeKey string,
-) (string, error) {
-	return db.cache.GetMessageTypeKeyValue(ctx, clientMsgID, sessionType, typeKey)
-}
-
-func (db *commonMsgDatabase) GetOneMessageAllReactionList(
-	ctx context.Context,
-	clientMsgID string,
-	sessionType int32,
-) (map[string]string, error) {
-	return db.cache.GetOneMessageAllReactionList(ctx, clientMsgID, sessionType)
-}
-
-func (db *commonMsgDatabase) DeleteOneMessageKey(
-	ctx context.Context,
-	clientMsgID string,
-	sessionType int32,
-	subKey string,
-) error {
-	return db.cache.DeleteOneMessageKey(ctx, clientMsgID, sessionType, subKey)
-}
-
-func (db *commonMsgDatabase) InsertOrUpdateReactionExtendMsgSet(
-	ctx context.Context,
-	conversationID string,
-	sessionType int32,
-	clientMsgID string,
-	msgFirstModifyTime int64,
-	reactionExtensions map[string]*sdkws.KeyValue,
-) error {
-	return db.extendMsgDatabase.InsertOrUpdateReactionExtendMsgSet(
-		ctx,
-		conversationID,
-		sessionType,
-		clientMsgID,
-		msgFirstModifyTime,
-		db.extendMsgSetModel.Pb2Model(reactionExtensions),
-	)
-}
-
-func (db *commonMsgDatabase) GetExtendMsg(
-	ctx context.Context,
-	conversationID string,
-	sessionType int32,
-	clientMsgID string,
-	maxMsgUpdateTime int64,
-) (*pbMsg.ExtendMsg, error) {
-	extendMsgSet, err := db.extendMsgDatabase.GetExtendMsgSet(ctx, conversationID, sessionType, maxMsgUpdateTime)
-	if err != nil {
-		return nil, err
-	}
-	extendMsg, ok := extendMsgSet.ExtendMsgs[clientMsgID]
-	if !ok {
-		return nil, errs.ErrRecordNotFound.Wrap(fmt.Sprintf("cant find client msg id: %s", clientMsgID))
-	}
-	reactionExtensionList := make(map[string]*pbMsg.KeyValueResp)
-	for key, model := range extendMsg.ReactionExtensionList {
-		reactionExtensionList[key] = &pbMsg.KeyValueResp{
-			KeyValue: &sdkws.KeyValue{
-				TypeKey:          model.TypeKey,
-				Value:            model.Value,
-				LatestUpdateTime: model.LatestUpdateTime,
-			},
-		}
-	}
-	return &pbMsg.ExtendMsg{
-		ReactionExtensions: reactionExtensionList,
-		ClientMsgID:        extendMsg.ClientMsgID,
-		MsgFirstModifyTime: extendMsg.MsgFirstModifyTime,
-		AttachedInfo:       extendMsg.AttachedInfo,
-		Ex:                 extendMsg.Ex,
-	}, nil
-}
-
-func (db *commonMsgDatabase) DeleteReactionExtendMsgSet(
-	ctx context.Context,
-	conversationID string,
-	sessionType int32,
-	clientMsgID string,
-	msgFirstModifyTime int64,
-	reactionExtensions map[string]*sdkws.KeyValue,
-) error {
-	return db.extendMsgDatabase.DeleteReactionExtendMsgSet(
-		ctx,
-		conversationID,
-		sessionType,
-		clientMsgID,
-		msgFirstModifyTime,
-		db.extendMsgSetModel.Pb2Model(reactionExtensions),
-	)
+func (db *commonMsgDatabase) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*unRelationTb.GroupCount, dateCount map[string]int64, err error) {
+	return db.msgDocDatabase.RangeGroupSendCount(ctx, start, end, ase, pageNumber, showNumber)
 }