feat: incremental synchronization of session list (#2408)

* fix: GroupApplicationAcceptedNotification

* fix: GroupApplicationAcceptedNotification

* fix: NotificationUserInfoUpdate

* cicd: robot automated Change

* fix: component

* fix: getConversationInfo

* feat: cron task

* feat: cron task

* feat: cron task

* feat: cron task

* feat: cron task

* fix: minio config url recognition error

* update gomake version

* update gomake version

* conversation version incremental

* GetOwnerConversation

* fix: change incremental syncer router name.

* fix: GetMsgDocModelByIndex bug

* update go.mod

---------

Co-authored-by: withchao <withchao@users.noreply.github.com>
Co-authored-by: Gordon <46924906+FGadvancer@users.noreply.github.com>

pkg/common/storage/cache/cachekey/conversation.go

@@ -23,6 +23,7 @@ const (
 	SuperGroupRecvMsgNotNotifyUserIDsKey     = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:"
 	SuperGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:"
 	ConversationNotReceiveMessageUserIDsKey  = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:"
+	ConversationUserMaxKey                   = "CONVERSATION_USER_MAX:"
 )
 
 func GetConversationKey(ownerUserID, conversationID string) string {
@@ -56,3 +57,7 @@ func GetConversationNotReceiveMessageUserIDsKey(conversationID string) string {
 func GetUserConversationIDsHashKey(ownerUserID string) string {
 	return ConversationIDsHashKey + ownerUserID
 }
+
+func GetConversationUserMaxVersionKey(userID string) string {
+	return ConversationUserMaxKey + userID
+}

pkg/common/storage/cache/conversation.go

@@ -54,4 +54,8 @@ type ConversationCache interface {
 
 	GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error)
 	DelConversationNotReceiveMessageUserIDs(conversationIDs ...string) ConversationCache
+
+	DelConversationVersionUserIDs(userIDs ...string) ConversationCache
+
+	FindMaxConversationUserVersion(ctx context.Context, userID string) (*relationtb.VersionLog, error)
 }

pkg/common/storage/cache/redis/conversation.go

@@ -95,6 +95,10 @@ func (c *ConversationRedisCache) getUserConversationIDsHashKey(ownerUserID strin
 	return cachekey.GetUserConversationIDsHashKey(ownerUserID)
 }
 
+func (c *ConversationRedisCache) getConversationUserMaxVersionKey(ownerUserID string) string {
+	return cachekey.GetConversationUserMaxVersionKey(ownerUserID)
+}
+
 func (c *ConversationRedisCache) GetUserConversationIDs(ctx context.Context, ownerUserID string) ([]string, error) {
 	return getCache(ctx, c.rcClient, c.getConversationIDsKey(ownerUserID), c.expireTime, func(ctx context.Context) ([]string, error) {
 		return c.conversationDB.FindUserIDAllConversationID(ctx, ownerUserID)
@@ -233,6 +237,19 @@ func (c *ConversationRedisCache) DelConversationNotReceiveMessageUserIDs(convers
 	for _, conversationID := range conversationIDs {
 		cache.AddKeys(c.getConversationNotReceiveMessageUserIDsKey(conversationID))
 	}
-
 	return cache
 }
+
+func (c *ConversationRedisCache) DelConversationVersionUserIDs(userIDs ...string) cache.ConversationCache {
+	cache := c.CloneConversationCache()
+	for _, userID := range userIDs {
+		cache.AddKeys(c.getConversationUserMaxVersionKey(userID))
+	}
+	return cache
+}
+
+func (c *ConversationRedisCache) FindMaxConversationUserVersion(ctx context.Context, userID string) (*model.VersionLog, error) {
+	return getCache(ctx, c.rcClient, c.getConversationUserMaxVersionKey(userID), c.expireTime, func(ctx context.Context) (*model.VersionLog, error) {
+		return c.conversationDB.FindConversationUserVersion(ctx, userID, 0, 0)
+	})
+}

pkg/common/storage/controller/conversation.go

@@ -66,6 +66,9 @@ type ConversationDatabase interface {
 	GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error)
 	// GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error)
 	// FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error)
+	FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*relationtb.VersionLog, error)
+	FindMaxConversationUserVersionCache(ctx context.Context, userID string) (*relationtb.VersionLog, error)
+	GetOwnerConversation(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relationtb.Conversation, error)
 }
 
 func NewConversationDatabase(conversation database.Conversation, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase {
@@ -106,6 +109,7 @@ func (c *conversationDatabase) SetUsersConversationFieldTx(ctx context.Context,
 			if _, ok := fieldMap["recv_msg_opt"]; ok {
 				cache = cache.DelConversationNotReceiveMessageUserIDs(conversation.ConversationID)
 			}
+			cache = cache.DelConversationVersionUserIDs(haveUserIDs...)
 		}
 		NotUserIDs := stringutil.DifferenceString(haveUserIDs, userIDs)
 		log.ZDebug(ctx, "SetUsersConversationFieldTx", "NotUserIDs", NotUserIDs, "haveUserIDs", haveUserIDs, "userIDs", userIDs)
@@ -137,7 +141,7 @@ func (c *conversationDatabase) UpdateUsersConversationField(ctx context.Context,
 		return err
 	}
 	cache := c.cache.CloneConversationCache()
-	cache = cache.DelUsersConversation(conversationID, userIDs...)
+	cache = cache.DelUsersConversation(conversationID, userIDs...).DelConversationVersionUserIDs(userIDs...)
 	if _, ok := args["recv_msg_opt"]; ok {
 		cache = cache.DelConversationNotReceiveMessageUserIDs(conversationID)
 	}
@@ -155,13 +159,14 @@ func (c *conversationDatabase) CreateConversation(ctx context.Context, conversat
 		cache = cache.DelConversationNotReceiveMessageUserIDs(conversation.ConversationID)
 		userIDs = append(userIDs, conversation.OwnerUserID)
 	}
-	return cache.DelConversationIDs(userIDs...).DelUserConversationIDsHash(userIDs...).ChainExecDel(ctx)
+	return cache.DelConversationIDs(userIDs...).DelUserConversationIDsHash(userIDs...).DelConversationVersionUserIDs(userIDs...).ChainExecDel(ctx)
 }
 
 func (c *conversationDatabase) SyncPeerUserPrivateConversationTx(ctx context.Context, conversations []*relationtb.Conversation) error {
 	return c.tx.Transaction(ctx, func(ctx context.Context) error {
 		cache := c.cache.CloneConversationCache()
 		for _, conversation := range conversations {
+			cache = cache.DelConversationVersionUserIDs(conversation.OwnerUserID)
 			for _, v := range [][2]string{{conversation.OwnerUserID, conversation.UserID}, {conversation.UserID, conversation.OwnerUserID}} {
 				ownerUserID := v[0]
 				userID := v[1]
@@ -207,6 +212,7 @@ func (c *conversationDatabase) GetUserAllConversation(ctx context.Context, owner
 func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUserID string, conversations []*relationtb.Conversation) error {
 	return c.tx.Transaction(ctx, func(ctx context.Context) error {
 		cache := c.cache.CloneConversationCache()
+		cache = cache.DelConversationVersionUserIDs(ownerUserID)
 		groupIDs := datautil.Distinct(datautil.Filter(conversations, func(e *relationtb.Conversation) (string, bool) {
 			return e.GroupID, e.GroupID != ""
 		}))
@@ -322,3 +328,28 @@ func (c *conversationDatabase) GetConversationIDsNeedDestruct(ctx context.Contex
 func (c *conversationDatabase) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) {
 	return c.cache.GetConversationNotReceiveMessageUserIDs(ctx, conversationID)
 }
+
+func (c *conversationDatabase) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*relationtb.VersionLog, error) {
+	return c.conversationDB.FindConversationUserVersion(ctx, userID, version, limit)
+}
+
+func (c *conversationDatabase) FindMaxConversationUserVersionCache(ctx context.Context, userID string) (*relationtb.VersionLog, error) {
+	return c.cache.FindMaxConversationUserVersion(ctx, userID)
+}
+
+func (c *conversationDatabase) GetOwnerConversation(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relationtb.Conversation, error) {
+	conversationIDs, err := c.cache.GetUserConversationIDs(ctx, ownerUserID)
+	if err != nil {
+		return 0, nil, err
+	}
+	findConversationIDs := datautil.Paginate(conversationIDs, int(pagination.GetPageNumber()), int(pagination.GetShowNumber()))
+	conversations := make([]*relationtb.Conversation, 0, len(findConversationIDs))
+	for _, conversationID := range findConversationIDs {
+		conversation, err := c.cache.GetConversation(ctx, ownerUserID, conversationID)
+		if err != nil {
+			return 0, nil, err
+		}
+		conversations = append(conversations, conversation)
+	}
+	return int64(len(conversationIDs)), conversations, nil
+}

pkg/common/storage/database/conversation.go

@@ -22,7 +22,6 @@ import (
 
 type Conversation interface {
 	Create(ctx context.Context, conversations []*model.Conversation) (err error)
-	Delete(ctx context.Context, groupIDs []string) (err error)
 	UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (rows int64, err error)
 	Update(ctx context.Context, conversation *model.Conversation) (err error)
 	Find(ctx context.Context, ownerUserID string, conversationIDs []string) (conversations []*model.Conversation, err error)
@@ -39,4 +38,5 @@ type Conversation interface {
 	GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*model.Conversation, error)
 	GetConversationIDsNeedDestruct(ctx context.Context) ([]*model.Conversation, error)
 	GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error)
+	FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error)
 }

pkg/common/storage/database/mgo/conversation.go

@@ -41,40 +41,71 @@ func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) {
 	if err != nil {
 		return nil, errs.Wrap(err)
 	}
-	return &ConversationMgo{coll: coll}, nil
+	version, err := NewVersionLog(db.Collection(database.ConversationVersionName))
+	if err != nil {
+		return nil, err
+	}
+	return &ConversationMgo{version: version, coll: coll}, nil
 }
 
 type ConversationMgo struct {
-	coll *mongo.Collection
+	version database.VersionLog
+	coll    *mongo.Collection
 }
 
 func (c *ConversationMgo) Create(ctx context.Context, conversations []*model.Conversation) (err error) {
-	return mongoutil.InsertMany(ctx, c.coll, conversations)
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.InsertMany(ctx, c.coll, conversations)
+	}, func() error {
+		userConversation := make(map[string][]string)
+		for _, conversation := range conversations {
+			userConversation[conversation.OwnerUserID] = append(userConversation[conversation.OwnerUserID], conversation.ConversationID)
+		}
+		for userID, conversationIDs := range userConversation {
+			if err := c.version.IncrVersion(ctx, userID, conversationIDs, model.VersionStateInsert); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
 }
 
-func (c *ConversationMgo) Delete(ctx context.Context, groupIDs []string) (err error) {
-	return mongoutil.DeleteMany(ctx, c.coll, bson.M{"group_id": bson.M{"$in": groupIDs}})
-}
-
-func (c *ConversationMgo) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (rows int64, err error) {
-	if len(args) == 0 {
+func (c *ConversationMgo) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (int64, error) {
+	if len(args) == 0 || len(userIDs) == 0 {
 		return 0, nil
 	}
 	filter := bson.M{
 		"conversation_id": conversationID,
+		"owner_user_id":   bson.M{"$in": userIDs},
 	}
-	if len(userIDs) > 0 {
-		filter["owner_user_id"] = bson.M{"$in": userIDs}
-	}
-	res, err := mongoutil.UpdateMany(ctx, c.coll, filter, bson.M{"$set": args})
+	var rows int64
+	err := mongoutil.IncrVersion(func() error {
+		res, err := mongoutil.UpdateMany(ctx, c.coll, filter, bson.M{"$set": args})
+		if err != nil {
+			return err
+		}
+		rows = res.ModifiedCount
+		return nil
+	}, func() error {
+		for _, userID := range userIDs {
+			if err := c.version.IncrVersion(ctx, userID, []string{conversationID}, model.VersionStateUpdate); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
 	if err != nil {
 		return 0, err
 	}
-	return res.ModifiedCount, nil
+	return rows, nil
 }
 
 func (c *ConversationMgo) Update(ctx context.Context, conversation *model.Conversation) (err error) {
-	return mongoutil.UpdateOne(ctx, c.coll, bson.M{"owner_user_id": conversation.OwnerUserID, "conversation_id": conversation.ConversationID}, bson.M{"$set": conversation}, true)
+	return mongoutil.IncrVersion(func() error {
+		return mongoutil.UpdateOne(ctx, c.coll, bson.M{"owner_user_id": conversation.OwnerUserID, "conversation_id": conversation.ConversationID}, bson.M{"$set": conversation}, true)
+	}, func() error {
+		return c.version.IncrVersion(ctx, conversation.OwnerUserID, []string{conversation.ConversationID}, model.VersionStateUpdate)
+	})
 }
 
 func (c *ConversationMgo) Find(ctx context.Context, ownerUserID string, conversationIDs []string) (conversations []*model.Conversation, err error) {
@@ -178,3 +209,7 @@ func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Co
 		options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}),
 	)
 }
+
+func (c *ConversationMgo) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) {
+	return c.version.FindChangeLog(ctx, userID, version, limit)
+}

pkg/common/storage/database/mgo/msg.go

@@ -205,7 +205,7 @@ func (m *MsgMgo) GetMsgDocModelByIndex(ctx context.Context, conversationID strin
 	if sort != 1 && sort != -1 {
 		return nil, errs.ErrArgs.WrapMsg("mongo sort must be 1 or -1")
 	}
-	opt := options.Find().SetLimit(1).SetSkip(index).SetSort(bson.M{"doc_id": sort}).SetLimit(1)
+	opt := options.Find().SetSkip(index).SetSort(bson.M{"_id": sort}).SetLimit(1)
 	filter := bson.M{"doc_id": primitive.Regex{Pattern: fmt.Sprintf("^%s:", conversationID)}}
 	msgs, err := mongoutil.Find[*model.MsgDocModel](ctx, m.coll, filter, opt)
 	if err != nil {

pkg/common/storage/database/name.go

@@ -1,17 +1,18 @@
 package database
 
 const (
-	BlackName              = "black"
-	ConversationName       = "conversation"
-	FriendName             = "friend"
-	FriendVersionName      = "friend_version"
-	FriendRequestName      = "friend_request"
-	GroupName              = "group"
-	GroupMemberName        = "group_member"
-	GroupMemberVersionName = "group_member_version"
-	GroupJoinVersionName   = "group_join_version"
-	GroupRequestName       = "group_request"
-	LogName                = "log"
-	ObjectName             = "s3"
-	UserName               = "user"
+	BlackName               = "black"
+	ConversationName        = "conversation"
+	FriendName              = "friend"
+	FriendVersionName       = "friend_version"
+	FriendRequestName       = "friend_request"
+	GroupName               = "group"
+	GroupMemberName         = "group_member"
+	GroupMemberVersionName  = "group_member_version"
+	GroupJoinVersionName    = "group_join_version"
+	ConversationVersionName = "conversation_version"
+	GroupRequestName        = "group_request"
+	LogName                 = "log"
+	ObjectName              = "s3"
+	UserName                = "user"
 )