feat: Optimize Scheduled Task (#2985)

* pb

* fix: Modifying other fields while setting IsPrivateChat does not take effect

* fix: quote message error revoke

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

pkg/common/storage/controller/conversation.go

@@ -74,6 +74,8 @@ type ConversationDatabase interface {
 	GetNotNotifyConversationIDs(ctx context.Context, userID string) ([]string, error)
 	// GetPinnedConversationIDs gets pinned conversationIDs by userID
 	GetPinnedConversationIDs(ctx context.Context, userID string) ([]string, error)
+	// FindRandConversation finds random conversations based on the specified timestamp and limit.
+	FindRandConversation(ctx context.Context, ts int64, limit int) ([]*relationtb.Conversation, error)
 }
 
 func NewConversationDatabase(conversation database.Conversation, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase {
@@ -401,3 +403,7 @@ func (c *conversationDatabase) GetPinnedConversationIDs(ctx context.Context, use
 	}
 	return conversationIDs, nil
 }
+
+func (c *conversationDatabase) FindRandConversation(ctx context.Context, ts int64, limit int) ([]*relationtb.Conversation, error) {
+	return c.conversationDB.FindRandConversation(ctx, ts, limit)
+}

pkg/common/storage/controller/msg.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"strings"
 	"time"
 
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
@@ -69,6 +68,7 @@ type CommonMsgDatabase interface {
 	GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error)
 	GetMaxSeq(ctx context.Context, conversationID string) (int64, error)
 	SetMinSeqs(ctx context.Context, seqs map[string]int64) error
+	SetMinSeq(ctx context.Context, conversationID string, seq int64) error
 
 	SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error)
 	SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error
@@ -95,13 +95,16 @@ type CommonMsgDatabase interface {
 	ConvertMsgsDocLen(ctx context.Context, conversationIDs []string)
 
 	// get Msg when destruct msg before
-	GetBeforeMsg(ctx context.Context, ts int64, docIds []string, limit int) ([]*model.MsgDocModel, error)
-	DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error)
+	//DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error)
 
-	GetDocIDs(ctx context.Context) ([]string, error)
+	GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error)
 
 	SetUserConversationsMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error
 	SetUserConversationsMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error
+
+	DeleteDoc(ctx context.Context, docID string) error
+
+	GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error)
 }
 
 func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) {
@@ -806,9 +809,10 @@ func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID strin
 	return db.seqConversation.GetMaxSeq(ctx, conversationID)
 }
 
-func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error {
-	return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq)
-}
+//
+//func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error {
+//	return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq)
+//}
 
 func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error {
 	return db.seqConversation.SetMinSeqs(ctx, seqs)
@@ -947,56 +951,40 @@ func (db *commonMsgDatabase) ConvertMsgsDocLen(ctx context.Context, conversation
 	db.msgDocDatabase.ConvertMsgsDocLen(ctx, conversationIDs)
 }
 
-func (db *commonMsgDatabase) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) {
-	var msgs []*model.MsgDocModel
-	for i := 0; i < len(docIDs); i += 1000 {
-		end := i + 1000
-		if end > len(docIDs) {
-			end = len(docIDs)
-		}
-
-		res, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, docIDs[i:end], limit)
-		if err != nil {
-			return nil, err
-		}
-		msgs = append(msgs, res...)
-
-		if len(msgs) >= limit {
-			return msgs[:limit], nil
-		}
-	}
-	return msgs, nil
+func (db *commonMsgDatabase) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) {
+	return db.msgDocDatabase.GetRandBeforeMsg(ctx, ts, limit)
 }
 
-func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) {
-	var notNull int
-	index := make([]int, 0, len(doc.Msg))
-	for i, message := range doc.Msg {
-		if message.Msg != nil {
-			notNull++
-			if message.Msg.SendTime < ts {
-				index = append(index, i)
-			}
-		}
-	}
-	if len(index) == 0 {
-		return index, nil
-	}
-	maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq
-	conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")]
-	if err := db.setMinSeq(ctx, conversationID, maxSeq+1); err != nil {
-		return index, err
-	}
-	if len(index) == notNull {
-		log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq)
-		return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID)
-	} else {
-		log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq)
-		return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index)
-	}
-}
+//
+//func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) {
+//	var notNull int
+//	index := make([]int, 0, len(doc.Msg))
+//	for i, message := range doc.Msg {
+//		if message.Msg != nil {
+//			notNull++
+//			if message.Msg.SendTime < ts {
+//				index = append(index, i)
+//			}
+//		}
+//	}
+//	if len(index) == 0 {
+//		return index, nil
+//	}
+//	maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq
+//	conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")]
+//	if err := db.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil {
+//		return index, err
+//	}
+//	if len(index) == notNull {
+//		log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq)
+//		return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID)
+//	} else {
+//		log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq)
+//		return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index)
+//	}
+//}
 
-func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID string, seq int64) error {
+func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, seq int64) error {
 	dbSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID)
 	if err != nil {
 		if errors.Is(errs.Unwrap(err), redis.Nil) {
@@ -1010,8 +998,8 @@ func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID strin
 	return db.seqConversation.SetMinSeq(ctx, conversationID, seq)
 }
 
-func (db *commonMsgDatabase) GetDocIDs(ctx context.Context) ([]string, error) {
-	return db.msgDocDatabase.GetDocIDs(ctx)
+func (db *commonMsgDatabase) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) {
+	return db.msgDocDatabase.GetRandDocIDs(ctx, limit)
 }
 
 func (db *commonMsgDatabase) GetCacheMaxSeqWithTime(ctx context.Context, conversationIDs []string) (map[string]database.SeqTime, error) {
@@ -1026,3 +1014,11 @@ func (db *commonMsgDatabase) GetMaxSeqsWithTime(ctx context.Context, conversatio
 	// todo: only the time in the redis cache will be taken, not the message time
 	return db.seqConversation.GetMaxSeqsWithTime(ctx, conversationIDs)
 }
+
+func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error {
+	return db.msgDocDatabase.DeleteDoc(ctx, docID)
+}
+
+func (db *commonMsgDatabase) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) {
+	return db.msgDocDatabase.GetLastMessageSeqByTime(ctx, conversationID, time)
+}

pkg/common/storage/controller/s3.go

@@ -24,7 +24,6 @@ import (
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
 
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache"
-	"github.com/openimsdk/tools/db/pagination"
 	"github.com/openimsdk/tools/s3"
 	"github.com/openimsdk/tools/s3/cont"
 	"github.com/redis/go-redis/v9"
@@ -40,11 +39,10 @@ type S3Database interface {
 	SetObject(ctx context.Context, info *model.Object) error
 	StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error)
 	FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error)
-	FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error)
-	DeleteObject(ctx context.Context, name string) error
-	DeleteSpecifiedData(ctx context.Context, engine string, name string) error
-	FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error)
+	FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error)
+	DeleteSpecifiedData(ctx context.Context, engine string, name []string) error
 	DelS3Key(ctx context.Context, engine string, keys ...string) error
+	GetKeyCount(ctx context.Context, engine string, key string) (int64, error)
 }
 
 func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database {
@@ -120,21 +118,19 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf
 func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
 	return s.s3.FormData(ctx, name, size, contentType, duration)
 }
-func (s *s3Database) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) {
-	return s.db.FindNeedDeleteObjectByDB(ctx, duration, needDelType, pagination)
+
+func (s *s3Database) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) {
+	return s.db.FindExpirationObject(ctx, engine, expiration, needDelType, count)
 }
 
-func (s *s3Database) DeleteObject(ctx context.Context, name string) error {
-	return s.s3.DeleteObject(ctx, name)
+func (s *s3Database) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) {
+	return s.db.GetKeyCount(ctx, engine, key)
 }
-func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name string) error {
+
+func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name []string) error {
 	return s.db.Delete(ctx, engine, name)
 }
 
-func (s *s3Database) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) {
-	return s.db.FindModelsByKey(ctx, key)
-}
-
 func (s *s3Database) DelS3Key(ctx context.Context, engine string, keys ...string) error {
 	return s.s3cache.DelS3Key(ctx, engine, keys...)
 }

pkg/common/storage/database/conversation.go

@@ -42,4 +42,5 @@ type Conversation interface {
 	GetConversationIDsNeedDestruct(ctx context.Context) ([]*model.Conversation, error)
 	GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error)
 	FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error)
+	FindRandConversation(ctx context.Context, ts int64, limit int) ([]*model.Conversation, error)
 }

pkg/common/storage/database/mgo/conversation.go

@@ -228,3 +228,35 @@ func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Co
 func (c *ConversationMgo) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) {
 	return c.version.FindChangeLog(ctx, userID, version, limit)
 }
+
+func (c *ConversationMgo) FindRandConversation(ctx context.Context, ts int64, limit int) ([]*model.Conversation, error) {
+	pipeline := []bson.M{
+		{
+			"$match": bson.M{
+				"is_msg_destruct":   true,
+				"msg_destruct_time": bson.M{"$ne": 0},
+			},
+		},
+		{
+			"$addFields": bson.M{
+				"next_msg_destruct_timestamp": bson.M{
+					"$add": []any{
+						bson.M{
+							"$toLong": "$latest_msg_destruct_time",
+						}, "$msg_destruct_time"},
+				},
+			},
+		},
+		{
+			"$match": bson.M{
+				"next_msg_destruct_timestamp": bson.M{"$lt": ts},
+			},
+		},
+		{
+			"$sample": bson.M{
+				"size": limit,
+			},
+		},
+	}
+	return mongoutil.Aggregate[*model.Conversation](ctx, c.coll, pipeline)
+}

pkg/common/storage/database/mgo/msg.go

@@ -1227,8 +1227,7 @@ func (m *MsgMgo) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string
 	}
 }
 
-func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) {
-	limit := 5000
+func (m *MsgMgo) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) {
 	var skip int
 	var docIDs []string
 	var offset int
@@ -1267,15 +1266,18 @@ func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) {
 	return docIDs, errs.Wrap(err)
 }
 
-func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) {
+func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) {
 	return mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{
 		{
 			"$match": bson.M{
-				"doc_id": bson.M{
-					"$in": docIDs,
-				},
-				"msgs.msg.send_time": bson.M{
-					"$lt": ts,
+				"msgs": bson.M{
+					"$not": bson.M{
+						"$elemMatch": bson.M{
+							"msg.send_time": bson.M{
+								"$gt": ts,
+							},
+						},
+					},
 				},
 			},
 		},
@@ -1288,7 +1290,9 @@ func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, li
 			},
 		},
 		{
-			"$limit": limit,
+			"$sample": bson.M{
+				"size": limit,
+			},
 		},
 	})
 }
@@ -1305,53 +1309,58 @@ func (m *MsgMgo) DeleteMsgByIndex(ctx context.Context, docID string, index []int
 	return mongoutil.UpdateOne(ctx, m.coll, bson.M{"doc_id": docID}, bson.M{"$set": set}, true)
 }
 
-//func (m *MsgMgo) ClearMsg(ctx context.Context, t time.Time) (int64, error) {
-//	ts := t.UnixMilli()
-//	var count int64
-//	for {
-//		msgs, err := m.GetBeforeMsg(ctx, ts, 100)
-//		if err != nil {
-//			return count, err
-//		}
-//		if len(msgs) == 0 {
-//			return count, nil
-//		}
-//		for _, msg := range msgs {
-//			num, err := m.deleteOneMsg(ctx, ts, msg)
-//			count += num
-//			if err != nil {
-//				return count, err
-//			}
-//		}
-//	}
-//}
-
 func (m *MsgMgo) DeleteDoc(ctx context.Context, docID string) error {
 	return mongoutil.DeleteOne(ctx, m.coll, bson.M{"doc_id": docID})
 }
 
-//func (m *MsgMgo) DeleteDocMsg(ctx context.Context, ts int64, doc *relation.MsgDocModel) (int64, error) {
-//	var notNull int
-//	index := make([]int, 0, len(doc.Msg))
-//	for i, message := range doc.Msg {
-//		if message.Msg != nil {
-//			notNull++
-//			if message.Msg.SendTime < ts {
-//				index = append(index, i)
-//			}
-//		}
-//	}
-//	if len(index) == 0 {
-//		return 0, errs.New("no msg to delete").WrapMsg("deleteOneMsg", "docID", doc.DocID)
-//	}
-//	if len(index) == notNull {
-//		if err := m.DeleteDoc(ctx, doc.DocID); err != nil {
-//			return 0, err
-//		}
-//	} else {
-//		if err := m.setNullMsg(ctx, doc.DocID, index); err != nil {
-//			return 0, err
-//		}
-//	}
-//	return int64(len(index)), nil
-//}
+func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) {
+	pipeline := []bson.M{
+		{
+			"$match": bson.M{
+				"doc_id": bson.M{
+					"$regex": fmt.Sprintf("^%s", conversationID),
+				},
+			},
+		},
+		{
+			"$match": bson.M{
+				"msgs.msg.send_time": bson.M{
+					"$lte": time,
+				},
+			},
+		},
+		{
+			"$sort": bson.M{
+				"_id": -1,
+			},
+		},
+		{
+			"$limit": 1,
+		},
+		{
+			"$project": bson.M{
+				"_id":                0,
+				"doc_id":             1,
+				"msgs.msg.send_time": 1,
+				"msgs.msg.seq":       1,
+			},
+		},
+	}
+	res, err := mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, pipeline)
+	if err != nil {
+		return 0, err
+	}
+	if len(res) == 0 {
+		return 0, nil
+	}
+	var seq int64
+	for _, v := range res[0].Msg {
+		if v.Msg == nil {
+			continue
+		}
+		if v.Msg.SendTime <= time {
+			seq = v.Msg.Seq
+		}
+	}
+	return seq, nil
+}

pkg/common/storage/database/mgo/msg_test.go

@@ -3,12 +3,11 @@ package mgo
 import (
 	"context"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
-	"github.com/openimsdk/protocol/msg"
-	"github.com/openimsdk/protocol/sdkws"
 	"github.com/openimsdk/tools/db/mongoutil"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
+	"math"
 	"math/rand"
 	"strconv"
 	"testing"
@@ -16,35 +15,45 @@ import (
 )
 
 func TestName1(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
-	defer cancel()
-	cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)))
-
-	v := &MsgMgo{
-		coll: cli.Database("openim_v3").Collection("msg3"),
-	}
-
-	req := &msg.SearchMessageReq{
-		//RecvID: "3187706596",
-		//SendID:      "7009965934",
-		ContentType: 101,
-		//SendTime:    "2024-05-06",
-		//SessionType: 3,
-		Pagination: &sdkws.RequestPagination{
-			PageNumber: 1,
-			ShowNumber: 10,
-		},
-	}
-	total, res, err := v.SearchMessage(ctx, req)
-	if err != nil {
-		panic(err)
-	}
-
-	for i, re := range res {
-		t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content)
-	}
-
-	t.Log(total)
+	//ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
+	//defer cancel()
+	//cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)))
+	//
+	//v := &MsgMgo{
+	//	coll: cli.Database("openim_v3").Collection("msg3"),
+	//}
+	//
+	//req := &msg.SearchMessageReq{
+	//	//RecvID: "3187706596",
+	//	//SendID:      "7009965934",
+	//	ContentType: 101,
+	//	//SendTime:    "2024-05-06",
+	//	//SessionType: 3,
+	//	Pagination: &sdkws.RequestPagination{
+	//		PageNumber: 1,
+	//		ShowNumber: 10,
+	//	},
+	//}
+	//total, res, err := v.SearchMessage(ctx, req)
+	//if err != nil {
+	//	panic(err)
+	//}
+	//
+	//for i, re := range res {
+	//	t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content)
+	//}
+	//
+	//t.Log(total)
+	//
+	//msg, err := NewMsgMongo(cli.Database("openim_v3"))
+	//if err != nil {
+	//	panic(err)
+	//}
+	//res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000)
+	//if err != nil {
+	//	panic(err)
+	//}
+	//t.Log(len(res))
 }
 
 func TestName10(t *testing.T) {
@@ -73,3 +82,33 @@ func TestName10(t *testing.T) {
 	}
 
 }
+
+func TestName3(t *testing.T) {
+	t.Log(uint64(math.MaxUint64))
+	t.Log(int64(math.MaxInt64))
+
+	t.Log(int64(math.MinInt64))
+}
+
+func TestName4(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
+	defer cancel()
+	cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)))
+
+	msg, err := NewMsgMongo(cli.Database("openim_v3"))
+	if err != nil {
+		panic(err)
+	}
+	ts := time.Now().Add(-time.Hour * 24 * 5).UnixMilli()
+	t.Log(ts)
+	res, err := msg.GetLastMessageSeqByTime(ctx, "sg_1523453548", ts)
+	if err != nil {
+		panic(err)
+	}
+	t.Log(res)
+}
+
+func TestName5(t *testing.T) {
+	var v time.Time
+	t.Log(v.UnixMilli())
+}

pkg/common/storage/database/mgo/object.go

@@ -22,7 +22,6 @@ import (
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
 
 	"github.com/openimsdk/tools/db/mongoutil"
-	"github.com/openimsdk/tools/db/pagination"
 	"github.com/openimsdk/tools/errs"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -91,21 +90,25 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model.
 	return mongoutil.FindOne[*model.Object](ctx, o.coll, bson.M{"name": name, "engine": engine})
 }
 
-func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error {
-	return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine})
+func (o *S3Mongo) Delete(ctx context.Context, engine string, name []string) error {
+	if len(name) == 0 {
+		return nil
+	}
+	return mongoutil.DeleteOne(ctx, o.coll, bson.M{"engine": engine, "name": bson.M{"$in": name}})
 }
 
-// Find Expires object
-func (o *S3Mongo) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) {
-	return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{
-		"create_time": bson.M{"$lt": duration},
-		"group":       bson.M{"$in": needDelType},
-	}, pagination)
-}
-
-// Find object by key
-func (o *S3Mongo) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) {
+func (o *S3Mongo) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) {
+	opt := options.Find()
+	if count > 0 {
+		opt.SetLimit(count)
+	}
 	return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{
-		"key": key,
-	})
+		"engine":      engine,
+		"create_time": bson.M{"$lt": expiration},
+		"group":       bson.M{"$in": needDelType},
+	}, opt)
+}
+
+func (o *S3Mongo) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) {
+	return mongoutil.Count(ctx, o.coll, bson.M{"engine": engine, "key": key})
 }

pkg/common/storage/database/msg.go

@@ -45,7 +45,9 @@ type Msg interface {
 
 	DeleteDoc(ctx context.Context, docID string) error
 	DeleteMsgByIndex(ctx context.Context, docID string, index []int) error
-	GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error)
+	GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error)
 
-	GetDocIDs(ctx context.Context) ([]string, error)
+	GetRandDocIDs(ctx context.Context, limit int) ([]string, error)
+
+	GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error)
 }

pkg/common/storage/database/object.go

@@ -19,13 +19,12 @@ import (
 	"time"
 
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
-	"github.com/openimsdk/tools/db/pagination"
 )
 
 type ObjectInfo interface {
 	SetObject(ctx context.Context, obj *model.Object) error
 	Take(ctx context.Context, engine string, name string) (*model.Object, error)
-	Delete(ctx context.Context, engine string, name string) error
-	FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error)
-	FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error)
+	Delete(ctx context.Context, engine string, name []string) error
+	FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error)
+	GetKeyCount(ctx context.Context, engine string, key string) (int64, error)
 }