refactor: db cache batch refactor and batch consume message. (#2325)

* refactor: cmd update. * refactor: msg transfer refactor. * refactor: msg transfer refactor. * refactor: msg transfer refactor. * fix: read prometheus port when flag set to enable and prevent failure during startup. * fix: notification has counted unread counts bug fix. * fix: merge opensource code into local. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * refactor: delete message and message batch use lua. * fix: add protective measures against memory overflow.
2026-04-28 06:19:20 +08:00 · 2024-06-03 11:24:37 +08:00
parent 67fe13f089
commit 973442e3d3
31 changed files with 1147 additions and 1333 deletions
@@ -54,8 +54,6 @@ type CommonMsgDatabase interface {
 	MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, seqs []int64) error
 	// DeleteMessagesFromCache deletes message caches from Redis by sequence numbers.
 	DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error
-	// DelUserDeleteMsgsList deletes user's message deletion list.
-	DelUserDeleteMsgsList(ctx context.Context, conversationID string, seqs []int64)
 	// BatchInsertChat2Cache increments the sequence number and then batch inserts messages into the cache.
 	BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, err error)
 	// GetMsgBySeqsRange retrieves messages from MongoDB by a range of sequence numbers.
@@ -98,7 +96,6 @@ type CommonMsgDatabase interface {

 	// to mq
 	MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error
-	MsgToModifyMQ(ctx context.Context, key, conversarionID string, msgs []*sdkws.MsgData) error
 	MsgToPushMQ(ctx context.Context, key, conversarionID string, msg2mq *sdkws.MsgData) (int32, int64, error)
 	MsgToMongoMQ(ctx context.Context, key, conversarionID string, msgs []*sdkws.MsgData, lastSeq int64) error

@@ -150,14 +147,13 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cach
 //}

 type commonMsgDatabase struct {
-	msgDocDatabase   database.Msg
-	msgTable         model.MsgDocModel
-	msg              cache.MsgCache
-	seq              cache.SeqCache
-	producer         *kafka.Producer
-	producerToMongo  *kafka.Producer
-	producerToModify *kafka.Producer
-	producerToPush   *kafka.Producer
+	msgDocDatabase  database.Msg
+	msgTable        model.MsgDocModel
+	msg             cache.MsgCache
+	seq             cache.SeqCache
+	producer        *kafka.Producer
+	producerToMongo *kafka.Producer
+	producerToPush  *kafka.Producer
 }

 func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error {
@@ -165,14 +161,6 @@ func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sd
 	return err
 }

-func (db *commonMsgDatabase) MsgToModifyMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData) error {
-	if len(messages) > 0 {
-		_, _, err := db.producerToModify.SendMessage(ctx, key, &pbmsg.MsgDataToModifyByMQ{ConversationID: conversationID, Messages: messages})
-		return err
-	}
-	return nil
-}
-
 func (db *commonMsgDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) {
 	partition, offset, err := db.producerToPush.SendMessage(ctx, key, &pbmsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID})
 	if err != nil {
@@ -357,11 +345,7 @@ func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userI
 }

 func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error {
-	return db.msg.DeleteMessages(ctx, conversationID, seqs)
-}
-
-func (db *commonMsgDatabase) DelUserDeleteMsgsList(ctx context.Context, conversationID string, seqs []int64) {
-	db.msg.DelUserDeleteMsgsList(ctx, conversationID, seqs)
+	return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs)
 }

 func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) {
@@ -388,7 +372,7 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa
 		userSeqMap[m.SendID] = m.Seq
 	}

-	failedNum, err := db.msg.SetMessageToCache(ctx, conversationID, msgs)
+	failedNum, err := db.msg.SetMessagesToCache(ctx, conversationID, msgs)
 	if err != nil {
 		prommetrics.MsgInsertRedisFailedCounter.Add(float64(failedNum))
 		log.ZError(ctx, "setMessageToCache error", err, "len", len(msgs), "conversationID", conversationID)
@@ -584,6 +568,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
 	}
 	newBegin := seqs[0]
 	newEnd := seqs[len(seqs)-1]
+	var successMsgs []*sdkws.MsgData
 	log.ZDebug(ctx, "GetMsgBySeqsRange", "first seqs", seqs, "newBegin", newBegin, "newEnd", newEnd)
 	cachedMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, seqs)
 	if err != nil {
@@ -592,54 +577,12 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
 			log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", seqs)
 		}
 	}
-	var successMsgs []*sdkws.MsgData
-	if len(cachedMsgs) > 0 {
-		delSeqs, err := db.msg.GetUserDelList(ctx, userID, conversationID)
-		if err != nil && errs.Unwrap(err) != redis.Nil {
-			return 0, 0, nil, err
-		}
-		var cacheDelNum int
-		for _, msg := range cachedMsgs {
-			if !datautil.Contain(msg.Seq, delSeqs...) {
-				successMsgs = append(successMsgs, msg)
-			} else {
-				cacheDelNum += 1
-			}
-		}
-		log.ZDebug(ctx, "get delSeqs from redis", "delSeqs", delSeqs, "userID", userID, "conversationID", conversationID, "cacheDelNum", cacheDelNum)
-		var reGetSeqsCache []int64
-		for i := 1; i <= cacheDelNum; {
-			newSeq := newBegin - int64(i)
-			if newSeq >= begin {
-				if !datautil.Contain(newSeq, delSeqs...) {
-					log.ZDebug(ctx, "seq del in cache, a new seq in range append", "new seq", newSeq)
-					reGetSeqsCache = append(reGetSeqsCache, newSeq)
-					i++
-				}
-			} else {
-				break
-			}
-		}
-		if len(reGetSeqsCache) > 0 {
-			log.ZDebug(ctx, "reGetSeqsCache", "reGetSeqsCache", reGetSeqsCache)
-			cachedMsgs, failedSeqs2, err := db.msg.GetMessagesBySeq(ctx, conversationID, reGetSeqsCache)
-			if err != nil {
-				if err != redis.Nil {
-
-					log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", reGetSeqsCache)
-				}
-			}
-			failedSeqs = append(failedSeqs, failedSeqs2...)
-			successMsgs = append(successMsgs, cachedMsgs...)
-		}
-	}
-	log.ZDebug(ctx, "get msgs from cache", "successMsgs", successMsgs)
-	if len(failedSeqs) != 0 {
-		log.ZDebug(ctx, "msgs not exist in redis", "seqs", failedSeqs)
-	}
-	// get from cache or storage
+	successMsgs = append(successMsgs, cachedMsgs...)
+	log.ZDebug(ctx, "get msgs from cache", "cachedMsgs", cachedMsgs)
+	// get from cache or db

 	if len(failedSeqs) > 0 {
+		log.ZDebug(ctx, "msgs not exist in redis", "seqs", failedSeqs)
 		mongoMsgs, err := db.getMsgBySeqsRange(ctx, userID, conversationID, failedSeqs, begin, end)
 		if err != nil {

@@ -679,7 +622,7 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co
 			log.ZError(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID)
 		}
 	}
-	log.ZDebug(ctx, "storage.seq.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs",
+	log.ZDebug(ctx, "db.seq.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs",
 		seqs, "len(successMsgs)", len(successMsgs), "failedSeqs", failedSeqs)

 	if len(failedSeqs) > 0 {
@@ -705,12 +648,6 @@ func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Cont
 	if minSeq == 0 {
 		return nil
 	}
-	if remainTime == 0 {
-		err = db.msg.CleanUpOneConversationAllMsg(ctx, conversationID)
-		if err != nil {
-			log.ZWarn(ctx, "CleanUpOneUserAllMsg", err, "conversationID", conversationID)
-		}
-	}
 	return db.seq.SetMinSeq(ctx, conversationID, minSeq)
 }

@@ -830,7 +767,7 @@ func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversatio
 }

 func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, allSeqs []int64) error {
-	if err := db.msg.DeleteMessages(ctx, conversationID, allSeqs); err != nil {
+	if err := db.msg.DeleteMessagesFromCache(ctx, conversationID, allSeqs); err != nil {
 		return err
 	}
 	for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, allSeqs) {
@@ -846,21 +783,9 @@ func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conve
 }

 func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error {
-	cachedMsgs, _, err := db.msg.GetMessagesBySeq(ctx, conversationID, seqs)
-	if err != nil && errs.Unwrap(err) != redis.Nil {
-		log.ZWarn(ctx, "DeleteUserMsgsBySeqs", err, "conversationID", conversationID, "seqs", seqs)
+	if err := db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs); err != nil {
 		return err
 	}
-	if len(cachedMsgs) > 0 {
-		var cacheSeqs []int64
-		for _, msg := range cachedMsgs {
-			cacheSeqs = append(cacheSeqs, msg.Seq)
-		}
-		if err := db.msg.UserDeleteMsgs(ctx, conversationID, cacheSeqs, userID); err != nil {
-			return err
-		}
-	}
-
 	for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) {
 		for _, seq := range seqs {
 			if _, err := db.msgDocDatabase.PushUnique(ctx, docID, db.msgTable.GetMsgIndex(seq), "del_list", []string{userID}); err != nil {
@@ -1085,14 +1010,14 @@ func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, d
 	}
 }

-//func (storage *commonMsgDatabase) ClearMsg(ctx context.Context, ts int64) (err error) {
+//func (db *commonMsgDatabase) ClearMsg(ctx context.Context, ts int64) (err error) {
 //	var (
 //		docNum int
 //		msgNum int
 //		start  = time.Now()
 //	)
 //	for {
-//		msgs, err := storage.msgDocDatabase.GetBeforeMsg(ctx, ts, 100)
+//		msgs, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, 100)
 //		if err != nil {
 //			return err
 //		}
@@ -1100,7 +1025,7 @@ func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, d
 //			return nil
 //		}
 //		for _, msg := range msgs {
-//			num, err := storage.deleteOneMsg(ctx, ts, msg)
+//			num, err := db.deleteOneMsg(ctx, ts, msg)
 //			if err != nil {
 //				return err
 //			}