IM/open-im-server
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: 3.7.0 code conventions. (#2148)

Browse Source 
* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* feat: add code lint

* feat: add code lint

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Script Refactoring

* feat: code format

* Script Refactoring

* Script Refactoring

* Script Refactoring

* Adjust MinIO configuration settings

* Adjust configuration settings

* Adjust configuration settings

* refactor: config change.

* refactor: webhooks update.

* Adjust configuration settings

* refactor: webhooks update.

* Adjust configuration settings

* Adjust configuration settings

* Adjust configuration settings

* feat: s3 api addr

* refactor: webhooks update.

* Adjust configuration settings

* Adjust configuration settings

* Adjust configuration settings

* Adjust configuration settings

* Adjust configuration settings

* Adjust configuration settings

* Adjust configuration settings

* refactor: webhooks update.

* refactor: kafka update.

* Simplify the Docker Compose configuration, remove unnecessary environment variables, and eliminate the gateway service.

* refactor: kafka update.

* refactor: kafka update.

* Simplify the Docker Compose configuration, remove unnecessary environment variables, and eliminate the gateway service.

* Simplify the Docker Compose configuration, remove unnecessary environment variables, and eliminate the gateway service.

* Windows can compile and run.

* Windows can compile and run.

* refactor: kafka update.

* feat: msg cache split

* refactor: webhooks update

* refactor: webhooks update

* refactor: friends update

* refactor: group update

* refactor: third update

* refactor: api update

* refactor: crontab update

* refactor: msggateway update

* mage

* mage

* refactor: all module update.

* check

* refactor: all module update.

* load config

* load config

* load config

* load config

* refactor: all module update.

* refactor: all module update.

* refactor: all module update.

* refactor: all module update.

* refactor: all module update.

* Optimize Docker configuration and script.

* refactor: all module update.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* refactor: all module update.

* Optimize Docker configuration and script.

* refactor: all module update.

* refactor: all module update.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* update tools

* update tools

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* update protocol

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* refactor: all module update.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* Optimize Docker configuration and script.

* refactor: api remove token auth by redis directly.

* Code Refactoring

* refactor: websocket auth change to call rpc of auth.

* refactor: kick online user and remove token change to call auth rpc.

* refactor: kick online user and remove token change to call auth rpc.

* refactor: remove msggateway redis.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor webhook

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor webhook

* refactor: cmd update.

* refactor: cmd update.

* fix: runtime: goroutine stack exceeds

* refactor: cmd update.

* refactor notification

* refactor notification

* refactor

* refactor: cmd update.

* refactor: cmd update.

* refactor

* refactor

* refactor

* protojson

* protojson

* protojson

* go mod

* wrapperspb

* refactor: cmd update.

* refactor: cmd update.

* refactor: cmd update.

* refactor: context update.

* refactor: websocket update info.

* refactor: websocket update info.

* refactor: websocket update info.

* refactor: websocket update info.

* refactor: api name change.

* refactor: debug info.

* refactor: debug info.

* refactor: debug info.

* fix: update file

* refactor

* refactor

* refactor: debug info.

* refactor: debug info.

* refactor: debug info.

* refactor: debug info.

* refactor: debug info.

* refactor: debug info.

* fix: callback update.

* fix: callback update.

* refactor

* fix: update message.

* fix: msg cache timeout.

* refactor

* refactor

* fix: push update.

* fix: push update.

* fix: push update.

* fix: push update.

* fix: push update.

* fix: push update.

* fix: push update.

* fix: websocket handle error remove when upgrade error.

---------

Co-authored-by: skiffer-git <44203734@qq.com>
Co-authored-by: Xinwei Xiong (cubxxw) <3293172751nss@gmail.com>
Co-authored-by: withchao <993506633@qq.com>
This commit is contained in:
OpenIM-Gordon
2024-04-19 22:23:08 +08:00
committed by GitHub
parent cca5336a8a
commit b76816bc14
438 changed files with 11525 additions and 15033 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/common/db/controller/auth.go
+14 -10
View File
@@ -17,13 +17,12 @@ package controller
import (
"context"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/tokenverify"
"github.com/golang-jwt/jwt/v4"
"github.com/openimsdk/open-im-server/v3/pkg/authverify"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/tokenverify"
)
type AuthDatabase interface {
@@ -31,17 +30,18 @@ type AuthDatabase interface {
GetTokensWithoutError(ctx context.Context, userID string, platformID int) (map[string]int, error)
// Create token
CreateToken(ctx context.Context, userID string, platformID int) (string, error)
SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error
}
type authDatabase struct {
cache cache.MsgModel
cache cache.TokenModel
accessSecret string
accessExpire int64
config *config.GlobalConfig
}
func NewAuthDatabase(cache cache.MsgModel, accessSecret string, accessExpire int64, config *config.GlobalConfig) AuthDatabase {
return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire, config: config}
func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire int64) AuthDatabase {
return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire}
}
// If the result is empty.
@@ -49,6 +49,10 @@ func (a *authDatabase) GetTokensWithoutError(ctx context.Context, userID string,
return a.cache.GetTokensWithoutError(ctx, userID, platformID)
}
func (a *authDatabase) SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error {
return a.cache.SetTokenMapByUidPid(ctx, userID, platformID, m)
}
// Create Token.
func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformID int) (string, error) {
tokens, err := a.cache.GetTokensWithoutError(ctx, userID, platformID)
@@ -57,7 +61,7 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI
}
var deleteTokenKey []string
for k, v := range tokens {
_, err = tokenverify.GetClaimFromToken(k, authverify.Secret(a.config.Secret))
_, err = tokenverify.GetClaimFromToken(k, authverify.Secret(a.accessSecret))
if err != nil || v != constant.NormalToken {
deleteTokenKey = append(deleteTokenKey, k)
}
@@ -73,7 +77,7 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
tokenString, err := token.SignedString([]byte(a.accessSecret))
if err != nil {
return "", errs.Wrap(err, "token.SignedString")
return "", errs.WrapMsg(err, "token.SignedString")
}
return tokenString, a.cache.AddTokenFlag(ctx, userID, platformID, tokenString, constant.NormalToken)
}
pkg/common/db/controller/black.go
+4 -4
View File
@@ -17,11 +17,11 @@ package controller
import (
"context"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/utils/datautil"
)
type BlackDatabase interface {
@@ -86,7 +86,7 @@ func (b *blackDatabase) CheckIn(ctx context.Context, userID1, userID2 string) (i
return
}
log.ZDebug(ctx, "blackIDs", "user1BlackIDs", userID1BlackIDs, "user2BlackIDs", userID2BlackIDs)
return utils.IsContain(userID2, userID1BlackIDs), utils.IsContain(userID1, userID2BlackIDs), nil
return datautil.Contain(userID2, userID1BlackIDs...), datautil.Contain(userID1, userID2BlackIDs...), nil
}
// FindBlackIDs Get Blacklist List.
pkg/common/db/controller/conversation.go
+19 -18
View File
@@ -18,14 +18,15 @@ import (
"context"
"time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/db/tx"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/utils/datautil"
"github.com/openimsdk/tools/utils/stringutil"
)
type ConversationDatabase interface {
@@ -62,11 +63,11 @@ type ConversationDatabase interface {
GetConversationIDsNeedDestruct(ctx context.Context) ([]*relationtb.ConversationModel, error)
// GetConversationNotReceiveMessageUserIDs gets user IDs for users in a conversation who have not received messages.
GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error)
//GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error)
//FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error)
// GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error)
// FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error)
}
func NewConversationDatabase(conversation relationtb.ConversationModelInterface, cache cache.ConversationCache, tx tx.CtxTx) ConversationDatabase {
func NewConversationDatabase(conversation relationtb.ConversationModelInterface, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase {
return &conversationDatabase{
conversationDB: conversation,
cache: cache,
@@ -77,7 +78,7 @@ func NewConversationDatabase(conversation relationtb.ConversationModelInterface,
type conversationDatabase struct {
conversationDB relationtb.ConversationModelInterface
cache cache.ConversationCache
tx tx.CtxTx
tx tx.Tx
}
func (c *conversationDatabase) SetUsersConversationFieldTx(ctx context.Context, userIDs []string, conversation *relationtb.ConversationModel, fieldMap map[string]any) (err error) {
@@ -105,13 +106,13 @@ func (c *conversationDatabase) SetUsersConversationFieldTx(ctx context.Context,
cache = cache.DelConversationNotReceiveMessageUserIDs(conversation.ConversationID)
}
}
NotUserIDs := utils.DifferenceString(haveUserIDs, userIDs)
NotUserIDs := stringutil.DifferenceString(haveUserIDs, userIDs)
log.ZDebug(ctx, "SetUsersConversationFieldTx", "NotUserIDs", NotUserIDs, "haveUserIDs", haveUserIDs, "userIDs", userIDs)
var conversations []*relationtb.ConversationModel
now := time.Now()
for _, v := range NotUserIDs {
temp := new(relationtb.ConversationModel)
if err = utils.CopyStructFields(temp, conversation); err != nil {
if err = datautil.CopyStructFields(temp, conversation); err != nil {
return err
}
temp.OwnerUserID = v
@@ -205,7 +206,7 @@ func (c *conversationDatabase) GetUserAllConversation(ctx context.Context, owner
func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUserID string, conversations []*relationtb.ConversationModel) error {
return c.tx.Transaction(ctx, func(ctx context.Context) error {
cache := c.cache.NewCache()
groupIDs := utils.Distinct(utils.Filter(conversations, func(e *relationtb.ConversationModel) (string, bool) {
groupIDs := datautil.Distinct(datautil.Filter(conversations, func(e *relationtb.ConversationModel) (string, bool) {
return e.GroupID, e.GroupID != ""
}))
for _, groupID := range groupIDs {
@@ -235,7 +236,7 @@ func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUs
var notExistConversations []*relationtb.ConversationModel
for _, conversation := range conversations {
if !utils.IsContain(conversation.ConversationID, existConversationIDs) {
if !datautil.Contain(conversation.ConversationID, existConversationIDs...) {
notExistConversations = append(notExistConversations, conversation)
}
}
@@ -246,28 +247,28 @@ func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUs
}
cache = cache.DelConversationIDs(ownerUserID).
DelUserConversationIDsHash(ownerUserID).
DelConversationNotReceiveMessageUserIDs(utils.Slice(notExistConversations, func(e *relationtb.ConversationModel) string { return e.ConversationID })...)
DelConversationNotReceiveMessageUserIDs(datautil.Slice(notExistConversations, func(e *relationtb.ConversationModel) string { return e.ConversationID })...)
}
return cache.ExecDel(ctx)
})
}
//func (c *conversationDatabase) FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error) {
// func (c *conversationDatabase) FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error) {
// return c.cache.GetSuperGroupRecvMsgNotNotifyUserIDs(ctx, groupID)
//}
func (c *conversationDatabase) CreateGroupChatConversation(ctx context.Context, groupID string, userIDs []string) error {
return c.tx.Transaction(ctx, func(ctx context.Context) error {
cache := c.cache.NewCache()
conversationID := msgprocessor.GetConversationIDBySessionType(constant.SuperGroupChatType, groupID)
conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID)
existConversationUserIDs, err := c.conversationDB.FindUserID(ctx, userIDs, []string{conversationID})
if err != nil {
return err
}
notExistUserIDs := utils.DifferenceString(userIDs, existConversationUserIDs)
notExistUserIDs := stringutil.DifferenceString(userIDs, existConversationUserIDs)
var conversations []*relationtb.ConversationModel
for _, v := range notExistUserIDs {
conversation := relationtb.ConversationModel{ConversationType: constant.SuperGroupChatType, GroupID: groupID, OwnerUserID: v, ConversationID: conversationID}
conversation := relationtb.ConversationModel{ConversationType: constant.ReadGroupChatType, GroupID: groupID, OwnerUserID: v, ConversationID: conversationID}
conversations = append(conversations, &conversation)
cache = cache.DelConversations(v, conversationID).DelConversationNotReceiveMessageUserIDs(conversationID)
}
pkg/common/db/controller/doc.go
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright © 2023 OpenIM. All rights reserved.
// Copyright © 2024 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
pkg/common/db/controller/friend.go
+16 -17
View File
@@ -19,15 +19,15 @@ import (
"fmt"
"time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/mcontext"
"github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/db/tx"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/mcontext"
"github.com/openimsdk/tools/utils/datautil"
)
type FriendDatabase interface {
@@ -80,11 +80,11 @@ type FriendDatabase interface {
type friendDatabase struct {
friend relation.FriendModelInterface
friendRequest relation.FriendRequestModelInterface
tx tx.CtxTx
tx tx.Tx
cache cache.FriendCache
}
func NewFriendDatabase(friend relation.FriendModelInterface, friendRequest relation.FriendRequestModelInterface, cache cache.FriendCache, tx tx.CtxTx) FriendDatabase {
func NewFriendDatabase(friend relation.FriendModelInterface, friendRequest relation.FriendRequestModelInterface, cache cache.FriendCache, tx tx.Tx) FriendDatabase {
return &friendDatabase{friend: friend, friendRequest: friendRequest, cache: cache, tx: tx}
}
@@ -106,8 +106,8 @@ func (f *friendDatabase) CheckIn(ctx context.Context, userID1, userID2 string) (
}
// Check if userID2 is in userID1's friend list and vice versa
inUser1Friends = utils.IsContain(userID2, userID1FriendIDs)
inUser2Friends = utils.IsContain(userID1, userID2FriendIDs)
inUser1Friends = datautil.Contain(userID2, userID1FriendIDs...)
inUser2Friends = datautil.Contain(userID1, userID2FriendIDs...)
return inUser1Friends, inUser2Friends, nil
}
@@ -139,7 +139,7 @@ func (f *friendDatabase) AddFriendRequest(ctx context.Context, fromUserID, toUse
func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, addSource int32) (err error) {
return f.tx.Transaction(ctx, func(ctx context.Context) error {
cache := f.cache.NewCache()
// User find friends
// user find friends
fs1, err := f.friend.FindFriends(ctx, ownerUserID, friendUserIDs)
if err != nil {
return err
@@ -148,7 +148,7 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string,
for _, v := range friendUserIDs {
fs1 = append(fs1, &relation.FriendModel{OwnerUserID: ownerUserID, FriendUserID: v, AddSource: addSource, OperatorUserID: opUserID})
}
fs11 := utils.DistinctAny(fs1, func(e *relation.FriendModel) string {
fs11 := datautil.DistinctAny(fs1, func(e *relation.FriendModel) string {
return e.FriendUserID
})
@@ -165,7 +165,7 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string,
fs2 = append(fs2, &relation.FriendModel{OwnerUserID: v, FriendUserID: ownerUserID, AddSource: addSource, OperatorUserID: opUserID})
newFriendIDs = append(newFriendIDs, v)
}
fs22 := utils.DistinctAny(fs2, func(e *relation.FriendModel) string {
fs22 := datautil.DistinctAny(fs2, func(e *relation.FriendModel) string {
return e.OwnerUserID
})
err = f.friend.Create(ctx, fs22)
@@ -212,14 +212,13 @@ func (f *friendDatabase) RefuseFriendRequest(ctx context.Context, friendRequest
// AgreeFriendRequest accepts a friend request. It first checks for an existing, unprocessed request.
func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest *relation.FriendRequestModel) (err error) {
return f.tx.Transaction(ctx, func(ctx context.Context) error {
defer log.ZDebug(ctx, "return line")
now := time.Now()
fr, err := f.friendRequest.Take(ctx, friendRequest.FromUserID, friendRequest.ToUserID)
if err != nil {
return err
}
if fr.HandleResult != 0 {
return errs.ErrArgs.Wrap("the friend request has been processed")
return errs.ErrArgs.WrapMsg("the friend request has been processed")
}
friendRequest.HandlerUserID = mcontext.GetOpUserID(ctx)
friendRequest.HandleResult = constant.FriendResponseAgree
@@ -246,7 +245,7 @@ func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest *
if err != nil {
return err
}
existsMap := utils.SliceSet(utils.Slice(exists, func(friend *relation.FriendModel) [2]string {
existsMap := datautil.SliceSet(datautil.Slice(exists, func(friend *relation.FriendModel) [2]string {
return [...]string{friend.OwnerUserID, friend.FriendUserID} // My - Friend
}))
var adds []*relation.FriendModel
pkg/common/db/controller/group.go
+11 -9
View File
@@ -16,15 +16,16 @@ package controller
import (
"context"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/tools/utils"
"github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/db/tx"
"github.com/openimsdk/tools/utils/datautil"
"github.com/redis/go-redis/v9"
)
@@ -107,10 +108,11 @@ type GroupDatabase interface {
func NewGroupDatabase(
rdb redis.UniversalClient,
localCache *config.LocalCache,
groupDB relationtb.GroupModelInterface,
groupMemberDB relationtb.GroupMemberModelInterface,
groupRequestDB relationtb.GroupRequestModelInterface,
ctxTx tx.CtxTx,
ctxTx tx.Tx,
groupHash cache.GroupHash,
) GroupDatabase {
rcOptions := rockscache.NewDefaultOptions()
@@ -121,7 +123,7 @@ func NewGroupDatabase(
groupMemberDB: groupMemberDB,
groupRequestDB: groupRequestDB,
ctxTx: ctxTx,
cache: cache.NewGroupCacheRedis(rdb, groupDB, groupMemberDB, groupRequestDB, groupHash, rcOptions),
cache: cache.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash, rcOptions),
}
}
@@ -129,7 +131,7 @@ type groupDatabase struct {
groupDB relationtb.GroupModelInterface
groupMemberDB relationtb.GroupMemberModelInterface
groupRequestDB relationtb.GroupRequestModelInterface
ctxTx tx.CtxTx
ctxTx tx.Tx
cache cache.GroupCache
}
@@ -270,7 +272,7 @@ func (g *groupDatabase) PageGetJoinGroup(ctx context.Context, userID string, pag
if err != nil {
return 0, nil, err
}
for _, groupID := range utils.Paginate(groupIDs, int(pagination.GetPageNumber()), int(pagination.GetShowNumber())) {
for _, groupID := range datautil.Paginate(groupIDs, int(pagination.GetPageNumber()), int(pagination.GetShowNumber())) {
groupMembers, err := g.cache.GetGroupMembersInfo(ctx, groupID, []string{userID})
if err != nil {
return 0, nil, err
@@ -285,7 +287,7 @@ func (g *groupDatabase) PageGetGroupMember(ctx context.Context, groupID string,
if err != nil {
return 0, nil, err
}
pageIDs := utils.Paginate(groupMemberIDs, int(pagination.GetPageNumber()), int(pagination.GetShowNumber()))
pageIDs := datautil.Paginate(groupMemberIDs, int(pagination.GetPageNumber()), int(pagination.GetShowNumber()))
if len(pageIDs) == 0 {
return int64(len(groupMemberIDs)), nil, nil
}
pkg/common/db/controller/msg.go
+148 -193
View File
@@ -17,22 +17,21 @@ package controller
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/OpenIMSDK/protocol/constant"
pbmsg "github.com/OpenIMSDK/protocol/msg"
"github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/convert"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
unrelationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/common/kafka"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics"
"github.com/openimsdk/protocol/constant"
pbmsg "github.com/openimsdk/protocol/msg"
"github.com/openimsdk/protocol/sdkws"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/mq/kafka"
"github.com/openimsdk/tools/utils/datautil"
"github.com/openimsdk/tools/utils/timeutil"
"github.com/redis/go-redis/v9"
"go.mongodb.org/mongo-driver/mongo"
)
@@ -47,7 +46,7 @@ type CommonMsgDatabase interface {
// BatchInsertChat2DB inserts a batch of messages into the database for a specific conversation.
BatchInsertChat2DB(ctx context.Context, conversationID string, msgs []*sdkws.MsgData, currentMaxSeq int64) error
// RevokeMsg revokes a message in a conversation.
RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *unrelationtb.RevokeModel) error
RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *relation.RevokeModel) error
// MarkSingleChatMsgsAsRead marks messages as read for a single chat by sequence numbers.
MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, seqs []int64) error
// DeleteMessagesFromCache deletes message caches from Redis by sequence numbers.
@@ -100,75 +99,54 @@ type CommonMsgDatabase interface {
MsgToPushMQ(ctx context.Context, key, conversarionID string, msg2mq *sdkws.MsgData) (int32, int64, error)
MsgToMongoMQ(ctx context.Context, key, conversarionID string, msgs []*sdkws.MsgData, lastSeq int64) error
RangeUserSendCount(
ctx context.Context,
start time.Time,
end time.Time,
group bool,
ase bool,
pageNumber int32,
showNumber int32,
) (msgCount int64, userCount int64, users []*unrelationtb.UserCount, dateCount map[string]int64, err error)
RangeGroupSendCount(
ctx context.Context,
start time.Time,
end time.Time,
ase bool,
pageNumber int32,
showNumber int32,
) (msgCount int64, userCount int64, groups []*unrelationtb.GroupCount, dateCount map[string]int64, err error)
RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*relation.UserCount, dateCount map[string]int64, err error)
RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*relation.GroupCount, dateCount map[string]int64, err error)
ConvertMsgsDocLen(ctx context.Context, conversationIDs []string)
}
func NewCommonMsgDatabase(msgDocModel unrelationtb.MsgDocModelInterface, cacheModel cache.MsgModel, config *config.GlobalConfig) (CommonMsgDatabase, error) {
producerConfig := &kafka.ProducerConfig{
ProducerAck: config.Kafka.ProducerAck,
CompressType: config.Kafka.CompressType,
Username: config.Kafka.Username,
Password: config.Kafka.Password,
}
var tlsConfig *kafka.TLSConfig
if config.Kafka.TLS != nil {
tlsConfig = &kafka.TLSConfig{
CACrt: config.Kafka.TLS.CACrt,
ClientCrt: config.Kafka.TLS.ClientCrt,
ClientKey: config.Kafka.TLS.ClientKey,
ClientKeyPwd: config.Kafka.TLS.ClientKeyPwd,
InsecureSkipVerify: false,
}
}
producerToRedis, err := kafka.NewKafkaProducer(config.Kafka.Addr, config.Kafka.LatestMsgToRedis.Topic, producerConfig, tlsConfig)
func NewCommonMsgDatabase(msgDocModel relation.MsgDocModelInterface, msg cache.MsgCache, seq cache.SeqCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) {
conf, err := kafka.BuildProducerConfig(*kafkaConf.Build())
if err != nil {
return nil, err
}
producerToMongo, err := kafka.NewKafkaProducer(config.Kafka.Addr, config.Kafka.MsgToMongo.Topic, producerConfig, tlsConfig)
producerToRedis, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToRedisTopic)
if err != nil {
return nil, err
}
producerToPush, err := kafka.NewKafkaProducer(config.Kafka.Addr, config.Kafka.MsgToPush.Topic, producerConfig, tlsConfig)
producerToMongo, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToMongoTopic)
if err != nil {
return nil, err
}
producerToPush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToPushTopic)
if err != nil {
return nil, err
}
return &commonMsgDatabase{
msgDocDatabase: msgDocModel,
cache: cacheModel,
msg: msg,
seq: seq,
producer: producerToRedis,
producerToMongo: producerToMongo,
producerToPush: producerToPush,
}, nil
}
func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database, config *config.GlobalConfig) (CommonMsgDatabase, error) {
cacheModel := cache.NewMsgCacheModel(rdb, config)
msgDocModel := unrelation.NewMsgMongoDriver(database)
return NewCommonMsgDatabase(msgDocModel, cacheModel, config)
}
//func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database, config *tools.CronTaskConfig) (CommonMsgDatabase, error) {
// msgDocModel, err := mgo.NewMsgMongo(database)
// if err != nil {
// return nil, err
// }
// //todo MsgCacheTimeout
// msg := cache.NewMsgCache(rdb, 86400, config.RedisConfig.EnablePipeline)
// seq := cache.NewSeqCache(rdb)
// return NewCommonMsgDatabase(msgDocModel, msg, seq, &config.KafkaConfig)
//}
type commonMsgDatabase struct {
msgDocDatabase unrelationtb.MsgDocModelInterface
msg unrelationtb.MsgDocModel
cache cache.MsgModel
msgDocDatabase relation.MsgDocModelInterface
msgTable relation.MsgDocModel
msg cache.MsgCache
seq cache.SeqCache
producer *kafka.Producer
producerToMongo *kafka.Producer
producerToModify *kafka.Producer
@@ -209,24 +187,24 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI
if len(fields) == 0 {
return nil
}
num := db.msg.GetSingleGocMsgNum()
num := db.msgTable.GetSingleGocMsgNum()
// num = 100
for i, field := range fields { // Check the type of the field
var ok bool
switch key {
case updateKeyMsg:
var msg *unrelationtb.MsgDataModel
msg, ok = field.(*unrelationtb.MsgDataModel)
var msg *relation.MsgDataModel
msg, ok = field.(*relation.MsgDataModel)
if msg != nil && msg.Seq != firstSeq+int64(i) {
return errs.ErrInternalServer.Wrap("seq is invalid")
return errs.ErrInternalServer.WrapMsg("seq is invalid")
}
case updateKeyRevoke:
_, ok = field.(*unrelationtb.RevokeModel)
_, ok = field.(*relation.RevokeModel)
default:
return errs.ErrInternalServer.Wrap("key is invalid")
return errs.ErrInternalServer.WrapMsg("key is invalid")
}
if !ok {
return errs.ErrInternalServer.Wrap("field type is invalid")
return errs.ErrInternalServer.WrapMsg("field type is invalid")
}
}
// Returns true if the document exists in the database, false if the document does not exist in the database
@@ -235,8 +213,8 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI
res *mongo.UpdateResult
err error
)
docID := db.msg.GetDocID(conversationID, seq)
index := db.msg.GetMsgIndex(seq)
docID := db.msgTable.GetDocID(conversationID, seq)
index := db.msgTable.GetMsgIndex(seq)
field := fields[i]
switch key {
case updateKeyMsg:
@@ -261,31 +239,31 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI
continue // The current data has been updated, skip the current data
}
}
doc := unrelationtb.MsgDocModel{
DocID: db.msg.GetDocID(conversationID, seq),
Msg: make([]*unrelationtb.MsgInfoModel, num),
doc := relation.MsgDocModel{
DocID: db.msgTable.GetDocID(conversationID, seq),
Msg: make([]*relation.MsgInfoModel, num),
}
var insert int // Inserted data number
for j := i; j < len(fields); j++ {
seq = firstSeq + int64(j)
if db.msg.GetDocID(conversationID, seq) != doc.DocID {
if db.msgTable.GetDocID(conversationID, seq) != doc.DocID {
break
}
insert++
switch key {
case updateKeyMsg:
doc.Msg[db.msg.GetMsgIndex(seq)] = &unrelationtb.MsgInfoModel{
Msg: fields[j].(*unrelationtb.MsgDataModel),
doc.Msg[db.msgTable.GetMsgIndex(seq)] = &relation.MsgInfoModel{
Msg: fields[j].(*relation.MsgDataModel),
}
case updateKeyRevoke:
doc.Msg[db.msg.GetMsgIndex(seq)] = &unrelationtb.MsgInfoModel{
Revoke: fields[j].(*unrelationtb.RevokeModel),
doc.Msg[db.msgTable.GetMsgIndex(seq)] = &relation.MsgInfoModel{
Revoke: fields[j].(*relation.RevokeModel),
}
}
}
for i, model := range doc.Msg {
if model == nil {
model = &unrelationtb.MsgInfoModel{}
model = &relation.MsgInfoModel{}
doc.Msg[i] = model
}
if model.DelList == nil {
@@ -308,16 +286,16 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI
func (db *commonMsgDatabase) BatchInsertChat2DB(ctx context.Context, conversationID string, msgList []*sdkws.MsgData, currentMaxSeq int64) error {
if len(msgList) == 0 {
return errs.ErrArgs.Wrap("msgList is empty")
return errs.ErrArgs.WrapMsg("msgList is empty")
}
msgs := make([]any, len(msgList))
for i, msg := range msgList {
if msg == nil {
continue
}
var offlinePushModel *unrelationtb.OfflinePushModel
var offlinePushModel *relation.OfflinePushModel
if msg.OfflinePushInfo != nil {
offlinePushModel = &unrelationtb.OfflinePushModel{
offlinePushModel = &relation.OfflinePushModel{
Title: msg.OfflinePushInfo.Title,
Desc: msg.OfflinePushInfo.Desc,
Ex: msg.OfflinePushInfo.Ex,
@@ -325,7 +303,7 @@ func (db *commonMsgDatabase) BatchInsertChat2DB(ctx context.Context, conversatio
IOSBadgeCount: msg.OfflinePushInfo.IOSBadgeCount,
}
}
msgs[i] = &unrelationtb.MsgDataModel{
msgs[i] = &relation.MsgDataModel{
SendID: msg.SendID,
RecvID: msg.RecvID,
GroupID: msg.GroupID,
@@ -352,15 +330,15 @@ func (db *commonMsgDatabase) BatchInsertChat2DB(ctx context.Context, conversatio
return db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq)
}
func (db *commonMsgDatabase) RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *unrelationtb.RevokeModel) error {
func (db *commonMsgDatabase) RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *relation.RevokeModel) error {
return db.BatchInsertBlock(ctx, conversationID, []any{revoke}, updateKeyRevoke, seq)
}
func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, totalSeqs []int64) error {
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, totalSeqs) {
for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, totalSeqs) {
var indexes []int64
for _, seq := range seqs {
indexes = append(indexes, db.msg.GetMsgIndex(seq))
indexes = append(indexes, db.msgTable.GetMsgIndex(seq))
}
log.ZDebug(ctx, "MarkSingleChatMsgsAsRead", "userID", userID, "docID", docID, "indexes", indexes)
if err := db.msgDocDatabase.MarkSingleChatMsgsAsRead(ctx, userID, docID, indexes); err != nil {
@@ -372,25 +350,25 @@ func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userI
}
func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error {
return db.cache.DeleteMessages(ctx, conversationID, seqs)
return db.msg.DeleteMessages(ctx, conversationID, seqs)
}
func (db *commonMsgDatabase) DelUserDeleteMsgsList(ctx context.Context, conversationID string, seqs []int64) {
db.cache.DelUserDeleteMsgsList(ctx, conversationID, seqs)
db.msg.DelUserDeleteMsgsList(ctx, conversationID, seqs)
}
func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) {
currentMaxSeq, err := db.cache.GetMaxSeq(ctx, conversationID)
currentMaxSeq, err := db.seq.GetMaxSeq(ctx, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
log.ZError(ctx, "db.cache.GetMaxSeq", err)
log.ZError(ctx, "db.seq.GetMaxSeq", err)
return 0, false, err
}
lenList := len(msgs)
if int64(lenList) > db.msg.GetSingleGocMsgNum() {
return 0, false, errors.New("too large")
if int64(lenList) > db.msgTable.GetSingleGocMsgNum() {
return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap()
}
if lenList < 1 {
return 0, false, errors.New("too short as 0")
return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap()
}
if errs.Unwrap(err) == redis.Nil {
isNew = true
@@ -402,19 +380,22 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa
m.Seq = currentMaxSeq
userSeqMap[m.SendID] = m.Seq
}
failedNum, err := db.cache.SetMessageToCache(ctx, conversationID, msgs)
failedNum, err := db.msg.SetMessageToCache(ctx, conversationID, msgs)
if err != nil {
prommetrics.MsgInsertRedisFailedCounter.Add(float64(failedNum))
log.ZError(ctx, "setMessageToCache error", err, "len", len(msgs), "conversationID", conversationID)
} else {
prommetrics.MsgInsertRedisSuccessCounter.Inc()
}
err = db.cache.SetMaxSeq(ctx, conversationID, currentMaxSeq)
err = db.seq.SetMaxSeq(ctx, conversationID, currentMaxSeq)
if err != nil {
log.ZError(ctx, "db.cache.SetMaxSeq error", err, "conversationID", conversationID)
log.ZError(ctx, "db.seq.SetMaxSeq error", err, "conversationID", conversationID)
prommetrics.SeqSetFailedCounter.Inc()
}
err = db.cache.SetHasReadSeqs(ctx, conversationID, userSeqMap)
err = db.seq.SetHasReadSeqs(ctx, conversationID, userSeqMap)
if err != nil {
log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID)
prommetrics.SeqSetFailedCounter.Inc()
@@ -423,7 +404,7 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa
}
func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversationID string, seqs []int64) (totalMsgs []*sdkws.MsgData, err error) {
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, seqs) {
for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) {
// log.ZDebug(ctx, "getMsgBySeqs", "docID", docID, "seqs", seqs)
msgs, err := db.findMsgInfoBySeq(ctx, userID, docID, conversationID, seqs)
if err != nil {
@@ -436,7 +417,7 @@ func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversat
return totalMsgs, nil
}
func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][]*unrelationtb.MsgInfoModel, userID, conversationID string, msg *unrelationtb.MsgInfoModel) {
func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][]*relation.MsgInfoModel, userID, conversationID string, msg *relation.MsgInfoModel) {
if msg.IsRead {
msg.Msg.IsRead = true
}
@@ -458,12 +439,12 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][
if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.ContentType == constant.MsgRevokeNotification {
return
}
var msgs []*unrelationtb.MsgInfoModel
var msgs []*relation.MsgInfoModel
if v, ok := cache[quoteMsg.QuoteMessage.Seq]; ok {
msgs = v
} else {
if quoteMsg.QuoteMessage.Seq > 0 {
ms, err := db.msgDocDatabase.GetMsgBySeqIndexIn1Doc(ctx, userID, db.msg.GetDocID(conversationID, quoteMsg.QuoteMessage.Seq), []int64{quoteMsg.QuoteMessage.Seq})
ms, err := db.msgDocDatabase.GetMsgBySeqIndexIn1Doc(ctx, userID, db.msgTable.GetDocID(conversationID, quoteMsg.QuoteMessage.Seq), []int64{quoteMsg.QuoteMessage.Seq})
if err != nil {
log.ZError(ctx, "GetMsgBySeqIndexIn1Doc", err, "conversationID", conversationID, "seq", quoteMsg.QuoteMessage.Seq)
return
@@ -487,17 +468,17 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][
return
}
msg.Msg.Content = string(data)
if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msg.GetDocID(conversationID, msg.Msg.Seq), db.msg.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil {
if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msgTable.GetDocID(conversationID, msg.Msg.Seq), db.msgTable.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil {
log.ZError(ctx, "UpdateMsgContent", err)
}
}
func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, conversationID string, seqs []int64) (totalMsgs []*unrelationtb.MsgInfoModel, err error) {
func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, conversationID string, seqs []int64) (totalMsgs []*relation.MsgInfoModel, err error) {
msgs, err := db.msgDocDatabase.GetMsgBySeqIndexIn1Doc(ctx, userID, docID, seqs)
if err != nil {
return nil, err
}
tempCache := make(map[int64][]*unrelationtb.MsgInfoModel)
tempCache := make(map[int64][]*relation.MsgInfoModel)
for _, msg := range msgs {
db.handlerDBMsg(ctx, tempCache, userID, conversationID, msg)
}
@@ -506,7 +487,7 @@ func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID
func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID string, conversationID string, allSeqs []int64, begin, end int64) (seqMsgs []*sdkws.MsgData, err error) {
log.ZDebug(ctx, "getMsgBySeqsRange", "conversationID", conversationID, "allSeqs", allSeqs, "begin", begin, "end", end)
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, allSeqs) {
for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, allSeqs) {
log.ZDebug(ctx, "getMsgBySeqsRange", "docID", docID, "seqs", seqs)
msgs, err := db.findMsgInfoBySeq(ctx, userID, docID, conversationID, seqs)
if err != nil {
@@ -542,23 +523,23 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID strin
// "userMinSeq" can be set as the same value as the conversation's "maxSeq" at the moment they join the group.
// This ensures that their message retrieval starts from the point they joined.
func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) {
userMinSeq, err := db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
userMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
minSeq, err := db.cache.GetMinSeq(ctx, conversationID)
minSeq, err := db.seq.GetMinSeq(ctx, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
if userMinSeq > minSeq {
minSeq = userMinSeq
}
//"minSeq" represents the startSeq value that the user can retrieve.
// "minSeq" represents the startSeq value that the user can retrieve.
if minSeq > end {
log.ZInfo(ctx, "minSeq > end", "minSeq", minSeq, "end", end)
log.ZWarn(ctx, "minSeq > end", errs.New("minSeq>end"), "minSeq", minSeq, "end", end)
return 0, 0, nil, nil
}
maxSeq, err := db.cache.GetMaxSeq(ctx, conversationID)
maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
@@ -568,7 +549,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
maxSeq = userMaxSeq
}
}
//"maxSeq" represents the endSeq value that the user can retrieve.
// "maxSeq" represents the endSeq value that the user can retrieve.
if begin < minSeq {
begin = minSeq
@@ -576,9 +557,9 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
if end > maxSeq {
end = maxSeq
}
//"begin" and "end" represent the actual startSeq and endSeq values that the user can retrieve.
// "begin" and "end" represent the actual startSeq and endSeq values that the user can retrieve.
if end < begin {
return 0, 0, nil, errs.ErrArgs.Wrap("seq end < begin")
return 0, 0, nil, errs.ErrArgs.WrapMsg("seq end < begin")
}
var seqs []int64
if end-begin+1 <= num {
@@ -591,25 +572,13 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
}
}
//167 178 10
//if end-num < {
//
//}
//var seqs []int64
//for i := end; i > end-num; i-- {
// if i >= begin {
// seqs = append([]int64{i}, seqs...)
// } else {
// break
// }
//}
if len(seqs) == 0 {
return 0, 0, nil, nil
}
newBegin := seqs[0]
newEnd := seqs[len(seqs)-1]
log.ZDebug(ctx, "GetMsgBySeqsRange", "first seqs", seqs, "newBegin", newBegin, "newEnd", newEnd)
cachedMsgs, failedSeqs, err := db.cache.GetMessagesBySeq(ctx, conversationID, seqs)
cachedMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, seqs)
if err != nil {
if err != redis.Nil {
@@ -618,13 +587,13 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
}
var successMsgs []*sdkws.MsgData
if len(cachedMsgs) > 0 {
delSeqs, err := db.cache.GetUserDelList(ctx, userID, conversationID)
delSeqs, err := db.msg.GetUserDelList(ctx, userID, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
var cacheDelNum int
for _, msg := range cachedMsgs {
if !utils.Contain(msg.Seq, delSeqs...) {
if !datautil.Contain(msg.Seq, delSeqs...) {
successMsgs = append(successMsgs, msg)
} else {
cacheDelNum += 1
@@ -635,7 +604,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
for i := 1; i <= cacheDelNum; {
newSeq := newBegin - int64(i)
if newSeq >= begin {
if !utils.Contain(newSeq, delSeqs...) {
if !datautil.Contain(newSeq, delSeqs...) {
log.ZDebug(ctx, "seq del in cache, a new seq in range append", "new seq", newSeq)
reGetSeqsCache = append(reGetSeqsCache, newSeq)
i++
@@ -646,7 +615,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
}
if len(reGetSeqsCache) > 0 {
log.ZDebug(ctx, "reGetSeqsCache", "reGetSeqsCache", reGetSeqsCache)
cachedMsgs, failedSeqs2, err := db.cache.GetMessagesBySeq(ctx, conversationID, reGetSeqsCache)
cachedMsgs, failedSeqs2, err := db.msg.GetMessagesBySeq(ctx, conversationID, reGetSeqsCache)
if err != nil {
if err != redis.Nil {
@@ -676,15 +645,15 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
}
func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (int64, int64, []*sdkws.MsgData, error) {
userMinSeq, err := db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
userMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
minSeq, err := db.cache.GetMinSeq(ctx, conversationID)
minSeq, err := db.seq.GetMinSeq(ctx, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
maxSeq, err := db.cache.GetMaxSeq(ctx, conversationID)
maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
@@ -697,28 +666,14 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co
newSeqs = append(newSeqs, seq)
}
}
successMsgs, failedSeqs, err := db.cache.GetMessagesBySeq(ctx, conversationID, newSeqs)
successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs)
if err != nil {
if err != redis.Nil {
log.ZError(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID)
}
}
log.ZInfo(
ctx,
"db.cache.GetMessagesBySeq",
"userID",
userID,
"conversationID",
conversationID,
"seqs",
seqs,
"successMsgs",
len(successMsgs),
"failedSeqs",
failedSeqs,
"conversationID",
conversationID,
)
log.ZDebug(ctx, "db.seq.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs",
seqs, "len(successMsgs)", len(successMsgs), "failedSeqs", failedSeqs)
if len(failedSeqs) > 0 {
mongoMsgs, err := db.getMsgBySeqs(ctx, userID, conversationID, failedSeqs)
@@ -739,17 +694,17 @@ func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Cont
if err != nil {
return err
}
log.ZInfo(ctx, "DeleteConversationMsgsAndSetMinSeq", "conversationID", conversationID, "minSeq", minSeq)
log.ZDebug(ctx, "DeleteConversationMsgsAndSetMinSeq", "conversationID", conversationID, "minSeq", minSeq)
if minSeq == 0 {
return nil
}
if remainTime == 0 {
err = db.cache.CleanUpOneConversationAllMsg(ctx, conversationID)
err = db.msg.CleanUpOneConversationAllMsg(ctx, conversationID)
if err != nil {
log.ZWarn(ctx, "CleanUpOneUserAllMsg", err, "conversationID", conversationID)
}
}
return db.cache.SetMinSeq(ctx, conversationID, minSeq)
return db.seq.SetMinSeq(ctx, conversationID, minSeq)
}
func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error) {
@@ -759,7 +714,7 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string
msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1)
if err != nil || msgDocModel.DocID == "" {
if err != nil {
if err == unrelation.ErrMsgListNotExist {
if err == relation.ErrMsgListNotExist {
log.ZDebug(ctx, "not doc find", "conversationID", conversationID, "userID", userID, "index", index)
} else {
log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index)
@@ -769,14 +724,14 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string
break
}
index++
//&& msgDocModel.Msg[0].Msg.SendTime > lastMsgDestructTime.UnixMilli()
// && msgDocModel.Msg[0].Msg.SendTime > lastMsgDestructTime.UnixMilli()
if len(msgDocModel.Msg) > 0 {
i := 0
var over bool
for _, msg := range msgDocModel.Msg {
i++
if msg != nil && msg.Msg != nil && msg.Msg.SendTime+destructTime*1000 <= time.Now().UnixMilli() {
if msg.Msg.SendTime+destructTime*1000 > lastMsgDestructTime.UnixMilli() && !utils.Contain(userID, msg.DelList...) {
if msg.Msg.SendTime+destructTime*1000 > lastMsgDestructTime.UnixMilli() && !datautil.Contain(userID, msg.DelList...) {
seqs = append(seqs, msg.Msg.Seq)
}
} else {
@@ -794,12 +749,12 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string
log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversationID, "userID", userID, "seqs", seqs)
if len(seqs) > 0 {
userMinSeq := seqs[len(seqs)-1] + 1
currentUserMinSeq, err := db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
currentUserMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return nil, err
}
if currentUserMinSeq < userMinSeq {
if err := db.cache.SetConversationUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil {
if err := db.seq.SetConversationUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil {
return nil, err
}
}
@@ -826,7 +781,7 @@ func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversatio
msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1)
if err != nil || msgDocModel.DocID == "" {
if err != nil {
if err == unrelation.ErrMsgListNotExist {
if err == relation.ErrMsgListNotExist {
log.ZDebug(ctx, "deleteMsgRecursion ErrMsgListNotExist", "conversationID", conversationID, "index:", index)
} else {
log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index)
@@ -840,10 +795,10 @@ func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversatio
return delStruct.getSetMinSeq() + 1, nil
}
log.ZDebug(ctx, "doc info", "conversationID", conversationID, "index", index, "docID", msgDocModel.DocID, "len", len(msgDocModel.Msg))
if int64(len(msgDocModel.Msg)) > db.msg.GetSingleGocMsgNum() {
if int64(len(msgDocModel.Msg)) > db.msgTable.GetSingleGocMsgNum() {
log.ZWarn(ctx, "msgs too large", nil, "lenth", len(msgDocModel.Msg), "docID:", msgDocModel.DocID)
}
if msgDocModel.IsFull() && msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < utils.GetCurrentTimestampByMill() {
if msgDocModel.IsFull() && msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < timeutil.GetCurrentTimestampByMill() {
log.ZDebug(ctx, "doc is full and all msg is expired", "docID", msgDocModel.DocID)
delStruct.delDocIDs = append(delStruct.delDocIDs, msgDocModel.DocID)
delStruct.minSeq = msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.Seq
@@ -851,7 +806,7 @@ func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversatio
var delMsgIndexs []int
for i, MsgInfoModel := range msgDocModel.Msg {
if MsgInfoModel != nil && MsgInfoModel.Msg != nil {
if utils.GetCurrentTimestampByMill() > MsgInfoModel.Msg.SendTime+(remainTime*1000) {
if timeutil.GetCurrentTimestampByMill() > MsgInfoModel.Msg.SendTime+(remainTime*1000) {
delMsgIndexs = append(delMsgIndexs, i)
}
}
@@ -868,13 +823,13 @@ func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversatio
}
func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, allSeqs []int64) error {
if err := db.cache.DeleteMessages(ctx, conversationID, allSeqs); err != nil {
if err := db.msg.DeleteMessages(ctx, conversationID, allSeqs); err != nil {
return err
}
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, allSeqs) {
for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, allSeqs) {
var indexes []int
for _, seq := range seqs {
indexes = append(indexes, int(db.msg.GetMsgIndex(seq)))
indexes = append(indexes, int(db.msgTable.GetMsgIndex(seq)))
}
if err := db.msgDocDatabase.DeleteMsgsInOneDocByIndex(ctx, docID, indexes); err != nil {
return err
@@ -884,7 +839,7 @@ func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conve
}
func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error {
cachedMsgs, _, err := db.cache.GetMessagesBySeq(ctx, conversationID, seqs)
cachedMsgs, _, err := db.msg.GetMessagesBySeq(ctx, conversationID, seqs)
if err != nil && errs.Unwrap(err) != redis.Nil {
log.ZWarn(ctx, "DeleteUserMsgsBySeqs", err, "conversationID", conversationID, "seqs", seqs)
return err
@@ -894,14 +849,14 @@ func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID st
for _, msg := range cachedMsgs {
cacheSeqs = append(cacheSeqs, msg.Seq)
}
if err := db.cache.UserDeleteMsgs(ctx, conversationID, cacheSeqs, userID); err != nil {
if err := db.msg.UserDeleteMsgs(ctx, conversationID, cacheSeqs, userID); err != nil {
return err
}
}
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, seqs) {
for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) {
for _, seq := range seqs {
if _, err := db.msgDocDatabase.PushUnique(ctx, docID, db.msg.GetMsgIndex(seq), "del_list", []string{userID}); err != nil {
if _, err := db.msgDocDatabase.PushUnique(ctx, docID, db.msgTable.GetMsgIndex(seq), "del_list", []string{userID}); err != nil {
return err
}
}
@@ -915,91 +870,91 @@ func (db *commonMsgDatabase) DeleteMsgsBySeqs(ctx context.Context, conversationI
func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, user string, conversationIDs []string) {
for _, conversationID := range conversationIDs {
maxSeq, err := db.cache.GetMaxSeq(ctx, conversationID)
maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID)
if err != nil {
if err == redis.Nil {
log.ZInfo(ctx, "max seq is nil", "conversationID", conversationID)
log.ZDebug(ctx, "max seq is nil", "conversationID", conversationID)
} else {
log.ZError(ctx, "get max seq failed", err, "conversationID", conversationID)
}
continue
}
if err := db.cache.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil {
if err := db.seq.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil {
log.ZError(ctx, "set min seq failed", err, "conversationID", conversationID, "minSeq", maxSeq+1)
}
}
}
func (db *commonMsgDatabase) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error {
return db.cache.SetMaxSeq(ctx, conversationID, maxSeq)
return db.seq.SetMaxSeq(ctx, conversationID, maxSeq)
}
func (db *commonMsgDatabase) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) {
return db.cache.GetMaxSeqs(ctx, conversationIDs)
return db.seq.GetMaxSeqs(ctx, conversationIDs)
}
func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) {
return db.cache.GetMaxSeq(ctx, conversationID)
return db.seq.GetMaxSeq(ctx, conversationID)
}
func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error {
return db.cache.SetMinSeq(ctx, conversationID, minSeq)
return db.seq.SetMinSeq(ctx, conversationID, minSeq)
}
func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error {
return db.cache.SetMinSeqs(ctx, seqs)
return db.seq.SetMinSeqs(ctx, seqs)
}
func (db *commonMsgDatabase) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) {
return db.cache.GetMinSeqs(ctx, conversationIDs)
return db.seq.GetMinSeqs(ctx, conversationIDs)
}
func (db *commonMsgDatabase) GetMinSeq(ctx context.Context, conversationID string) (int64, error) {
return db.cache.GetMinSeq(ctx, conversationID)
return db.seq.GetMinSeq(ctx, conversationID)
}
func (db *commonMsgDatabase) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
return db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
return db.seq.GetConversationUserMinSeq(ctx, conversationID, userID)
}
func (db *commonMsgDatabase) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) {
return db.cache.GetConversationUserMinSeqs(ctx, conversationID, userIDs)
return db.seq.GetConversationUserMinSeqs(ctx, conversationID, userIDs)
}
func (db *commonMsgDatabase) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error {
return db.cache.SetConversationUserMinSeq(ctx, conversationID, userID, minSeq)
return db.seq.SetConversationUserMinSeq(ctx, conversationID, userID, minSeq)
}
func (db *commonMsgDatabase) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) {
return db.cache.SetConversationUserMinSeqs(ctx, conversationID, seqs)
return db.seq.SetConversationUserMinSeqs(ctx, conversationID, seqs)
}
func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error {
return db.cache.SetUserConversationsMinSeqs(ctx, userID, seqs)
return db.seq.SetUserConversationsMinSeqs(ctx, userID, seqs)
}
func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error {
return db.cache.UserSetHasReadSeqs(ctx, userID, hasReadSeqs)
return db.seq.UserSetHasReadSeqs(ctx, userID, hasReadSeqs)
}
func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error {
return db.cache.SetHasReadSeq(ctx, userID, conversationID, hasReadSeq)
return db.seq.SetHasReadSeq(ctx, userID, conversationID, hasReadSeq)
}
func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) {
return db.cache.GetHasReadSeqs(ctx, userID, conversationIDs)
return db.seq.GetHasReadSeqs(ctx, userID, conversationIDs)
}
func (db *commonMsgDatabase) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) {
return db.cache.GetHasReadSeq(ctx, userID, conversationID)
return db.seq.GetHasReadSeq(ctx, userID, conversationID)
}
func (db *commonMsgDatabase) SetSendMsgStatus(ctx context.Context, id string, status int32) error {
return db.cache.SetSendMsgStatus(ctx, id, status)
return db.msg.SetSendMsgStatus(ctx, id, status)
}
func (db *commonMsgDatabase) GetSendMsgStatus(ctx context.Context, id string) (int32, error) {
return db.cache.GetSendMsgStatus(ctx, id)
return db.msg.GetSendMsgStatus(ctx, id)
}
func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) {
@@ -1007,11 +962,11 @@ func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context
if err != nil {
return
}
minSeqCache, err = db.cache.GetMinSeq(ctx, conversationID)
minSeqCache, err = db.seq.GetMinSeq(ctx, conversationID)
if err != nil {
return
}
maxSeqCache, err = db.cache.GetMaxSeq(ctx, conversationID)
maxSeqCache, err = db.seq.GetMaxSeq(ctx, conversationID)
if err != nil {
return
}
@@ -1044,7 +999,7 @@ func (db *commonMsgDatabase) RangeUserSendCount(
ase bool,
pageNumber int32,
showNumber int32,
) (msgCount int64, userCount int64, users []*unrelationtb.UserCount, dateCount map[string]int64, err error) {
) (msgCount int64, userCount int64, users []*relation.UserCount, dateCount map[string]int64, err error) {
return db.msgDocDatabase.RangeUserSendCount(ctx, start, end, group, ase, pageNumber, showNumber)
}
@@ -1055,7 +1010,7 @@ func (db *commonMsgDatabase) RangeGroupSendCount(
ase bool,
pageNumber int32,
showNumber int32,
) (msgCount int64, userCount int64, groups []*unrelationtb.GroupCount, dateCount map[string]int64, err error) {
) (msgCount int64, userCount int64, groups []*relation.GroupCount, dateCount map[string]int64, err error) {
return db.msgDocDatabase.RangeGroupSendCount(ctx, start, end, ase, pageNumber, showNumber)
}
@@ -1078,12 +1033,12 @@ func (db *commonMsgDatabase) FindOneByDocIDs(ctx context.Context, conversationID
totalMsgs := make(map[string]*sdkws.MsgData)
for _, conversationID := range conversationIDs {
seq := seqs[conversationID]
docID := db.msg.GetDocID(conversationID, seq)
docID := db.msgTable.GetDocID(conversationID, seq)
msgs, err := db.msgDocDatabase.FindOneByDocID(ctx, docID)
if err != nil {
return nil, err
}
index := db.msg.GetMsgIndex(seq)
index := db.msgTable.GetMsgIndex(seq)
totalMsgs[conversationID] = convert.MsgDB2Pb(msgs.Msg[index].Msg)
}
return totalMsgs, nil
pkg/common/db/controller/msg_test.go
-250
View File
@@ -1,250 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controller
import (
"context"
"fmt"
"math/rand"
"strconv"
"sync"
"testing"
"time"
"github.com/OpenIMSDK/tools/log"
"go.mongodb.org/mongo-driver/bson"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
unrelationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
)
func Test_BatchInsertChat2DB(t *testing.T) {
conf := config.NewGlobalConfig()
conf.Mongo.Address = []string{"192.168.44.128:37017"}
// conf.Mongo.Timeout = 60
conf.Mongo.Database = "openIM"
// conf.Mongo.Source = "admin"
conf.Mongo.Username = "root"
conf.Mongo.Password = "openIM123"
conf.Mongo.MaxPoolSize = 100
conf.RetainChatRecords = 3650
conf.ChatRecordsClearTime = "0 2 * * 3"
mongo, err := unrelation.NewMongo(conf)
if err != nil {
t.Fatal(err)
}
err = mongo.GetDatabase(conf.Mongo.Database).Client().Ping(context.Background(), nil)
if err != nil {
panic(err)
}
db := &commonMsgDatabase{
msgDocDatabase: unrelation.NewMsgMongoDriver(mongo.GetDatabase(conf.Mongo.Database)),
}
//ctx := context.Background()
//msgs := make([]*sdkws.MsgData, 0, 1)
//for i := 0; i < cap(msgs); i++ {
// msgs = append(msgs, &sdkws.MsgData{
// Content: []byte(fmt.Sprintf("test-%d", i)),
// SendTime: time.Now().UnixMilli(),
// })
//}
//err = db.BatchInsertChat2DB(ctx, "test", msgs, 0)
//if err != nil {
// panic(err)
//}
_ = db.BatchInsertChat2DB
c := mongo.GetDatabase(conf.Mongo.Database).Collection("msg")
ch := make(chan int)
rand.Seed(time.Now().UnixNano())
index := 10
var wg sync.WaitGroup
for i := 0; i < 1000; i++ {
wg.Add(1)
go func(channelID int) {
defer wg.Done()
<-ch
var arr []string
for i := 0; i < 500; i++ {
arr = append(arr, strconv.Itoa(i+1))
}
rand.Shuffle(len(arr), func(i, j int) {
arr[i], arr[j] = arr[j], arr[i]
})
for j, s := range arr {
if j == 0 {
fmt.Printf("channnelID: %d, arr[0]: %s\n", channelID, arr[j])
}
filter := bson.M{"doc_id": "test:0"}
update := bson.M{
"$addToSet": bson.M{
fmt.Sprintf("msgs.%d.del_list", index): bson.M{"$each": []string{s}},
},
}
_, err := c.UpdateOne(context.Background(), filter, update)
if err != nil {
t.Fatal(err)
}
}
}(i)
}
for i := 0; i < 1000; i++ {
wg.Add(1)
go func() {
defer wg.Done()
<-ch
var arr []string
for i := 0; i < 500; i++ {
arr = append(arr, strconv.Itoa(1001+i))
}
rand.Shuffle(len(arr), func(i, j int) {
arr[i], arr[j] = arr[j], arr[i]
})
for _, s := range arr {
filter := bson.M{"doc_id": "test:0"}
update := bson.M{
"$addToSet": bson.M{
fmt.Sprintf("msgs.%d.read_list", index): bson.M{"$each": []string{s}},
},
}
_, err := c.UpdateOne(context.Background(), filter, update)
if err != nil {
t.Fatal(err)
}
}
}()
}
time.Sleep(time.Second * 2)
close(ch)
wg.Wait()
}
func GetDB() *commonMsgDatabase {
conf := config.NewGlobalConfig()
conf.Mongo.Address = []string{"203.56.175.233:37017"}
// conf.Mongo.Timeout = 60
conf.Mongo.Database = "openim_v3"
// conf.Mongo.Source = "admin"
conf.Mongo.Username = "root"
conf.Mongo.Password = "openIM123"
conf.Mongo.MaxPoolSize = 100
conf.RetainChatRecords = 3650
conf.ChatRecordsClearTime = "0 2 * * 3"
mongo, err := unrelation.NewMongo(conf)
if err != nil {
panic(err)
}
err = mongo.GetDatabase(conf.Mongo.Database).Client().Ping(context.Background(), nil)
if err != nil {
panic(err)
}
return &commonMsgDatabase{
msgDocDatabase: unrelation.NewMsgMongoDriver(mongo.GetDatabase(conf.Mongo.Database)),
}
}
func Test_Insert(t *testing.T) {
db := GetDB()
ctx := context.Background()
var arr []any
for i := 0; i < 345; i++ {
if i%2 == 0 {
arr = append(arr, (*unrelationtb.MsgDataModel)(nil))
continue
}
arr = append(arr, &unrelationtb.MsgDataModel{
Seq: int64(i),
Content: fmt.Sprintf("test-%d", i),
})
}
if err := db.BatchInsertBlock(ctx, "test", arr, updateKeyMsg, 1); err != nil {
t.Fatal(err)
}
}
func Test_Revoke(t *testing.T) {
db := GetDB()
ctx := context.Background()
var arr []any
for i := 0; i < 456; i++ {
arr = append(arr, &unrelationtb.RevokeModel{
UserID: "uid_" + strconv.Itoa(i),
Nickname: "uname_" + strconv.Itoa(i),
Time: time.Now().UnixMilli(),
})
}
if err := db.BatchInsertBlock(ctx, "test", arr, updateKeyRevoke, 123); err != nil {
t.Fatal(err)
}
}
func Test_FindBySeq(t *testing.T) {
if err := log.InitFromConfig("", "", 6, true, false, "", 2, 1); err != nil {
t.Fatal(err)
}
db := GetDB()
ctx := context.Background()
fmt.Println(
db.msgDocDatabase.(*unrelation.MsgMongoDriver).GetMsgBySeqIndexIn1Doc(ctx, "100", "si_100_101:0", []int64{1}),
)
//res, err := db.msgDocDatabase.GetMsgBySeqIndexIn1Doc(ctx, "123456", "test:0", []int64{1, 2, 3})
//if err != nil {
// t.Fatal(err)
//}
//db.GetMsgBySeqs(ctx, "100", "si_100_101:0", []int64{6})
//data, _ := json.Marshal(res)
//fmt.Println(string(data))
}
//func Test_Delete(t *testing.T) {
// db := GetDB()
// ctx := context.Background()
// var arr []any
// for i := 0; i < 123; i++ {
// arr = append(arr, []string{"uid_1", "uid_2"})
// }
// if err := db.BatchInsertBlock(ctx, "test", arr, updateKeyDel, 210); err != nil {
// t.Fatal(err)
// }
//}
func TestName(t *testing.T) {
db := GetDB()
var seqs []int64
for i := int64(1); i <= 50; i++ {
seqs = append(seqs, i)
}
msgs, err := db.getMsgBySeqsRange(context.Background(), "4931176757", "si_3866692501_4931176757", seqs, seqs[0], seqs[len(seqs)-1])
if err != nil {
t.Fatal(err)
}
t.Log(msgs)
}
pkg/common/db/controller/push.go
+2 -2
View File
@@ -25,10 +25,10 @@ type PushDatabase interface {
}
type pushDataBase struct {
cache cache.MsgModel
cache cache.ThirdCache
}
func NewPushDatabase(cache cache.MsgModel) PushDatabase {
func NewPushDatabase(cache cache.ThirdCache) PushDatabase {
return &pushDataBase{cache: cache}
}
pkg/common/db/controller/s3.go
+2 -2
View File
@@ -20,9 +20,9 @@ import (
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cont"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/s3"
"github.com/openimsdk/tools/s3/cont"
"github.com/redis/go-redis/v9"
)
pkg/common/db/controller/third.go
+3 -3
View File
@@ -18,9 +18,9 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/db/pagination"
)
type ThirdDatabase interface {
@@ -34,7 +34,7 @@ type ThirdDatabase interface {
}
type thirdDatabase struct {
cache cache.MsgModel
cache cache.ThirdCache
logdb relation.LogInterface
}
@@ -58,7 +58,7 @@ func (t *thirdDatabase) UploadLogs(ctx context.Context, logs []*relation.LogMode
return t.logdb.Create(ctx, logs)
}
func NewThirdDatabase(cache cache.MsgModel, logdb relation.LogInterface) ThirdDatabase {
func NewThirdDatabase(cache cache.ThirdCache, logdb relation.LogInterface) ThirdDatabase {
return &thirdDatabase{cache: cache, logdb: logdb}
}
pkg/common/db/controller/user.go
+21 -34
View File
@@ -16,16 +16,16 @@ package controller
import (
"context"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/db/tx"
"github.com/openimsdk/tools/utils/datautil"
"time"
"github.com/OpenIMSDK/protocol/user"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/protocol/user"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
unrelationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
)
type UserDatabase interface {
@@ -39,13 +39,11 @@ type UserDatabase interface {
FindNotification(ctx context.Context, level int64) (users []*relation.UserModel, err error)
// Create Insert multiple external guarantees that the userID is not repeated and does not exist in the db
Create(ctx context.Context, users []*relation.UserModel) (err error)
// Update update (non-zero value) external guarantee userID exists
//Update(ctx context.Context, user *relation.UserModel) (err error)
// UpdateByMap update (zero value) external guarantee userID exists
UpdateByMap(ctx context.Context, userID string, args map[string]any) (err error)
// FindUser
PageFindUser(ctx context.Context, level1 int64, level2 int64, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error)
//FindUser with keyword
// FindUser with keyword
PageFindUserWithKeyword(ctx context.Context, level1 int64, level2 int64, userID string, nickName string, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error)
// Page If not found, no error is returned
Page(ctx context.Context, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error)
@@ -74,7 +72,7 @@ type UserDatabase interface {
// SetUserStatus Set the user status and store the user status in redis
SetUserStatus(ctx context.Context, userID string, status, platformID int32) error
//CRUD user command
// CRUD user command
AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error
DeleteUserCommand(ctx context.Context, userID string, Type int32, UUID string) error
UpdateUserCommand(ctx context.Context, userID string, Type int32, UUID string, val map[string]any) error
@@ -83,19 +81,19 @@ type UserDatabase interface {
}
type userDatabase struct {
tx tx.CtxTx
tx tx.Tx
userDB relation.UserModelInterface
cache cache.UserCache
mongoDB unrelationtb.UserModelInterface
mongoDB relation.SubscribeUserModelInterface
}
func NewUserDatabase(userDB relation.UserModelInterface, cache cache.UserCache, tx tx.CtxTx, mongoDB unrelationtb.UserModelInterface) UserDatabase {
func NewUserDatabase(userDB relation.UserModelInterface, cache cache.UserCache, tx tx.Tx, mongoDB relation.SubscribeUserModelInterface) UserDatabase {
return &userDatabase{userDB: userDB, cache: cache, tx: tx, mongoDB: mongoDB}
}
func (u *userDatabase) InitOnce(ctx context.Context, users []*relation.UserModel) error {
// Extract user IDs from the given user models.
userIDs := utils.Slice(users, func(e *relation.UserModel) string {
userIDs := datautil.Slice(users, func(e *relation.UserModel) string {
return e.UserID
})
@@ -106,7 +104,7 @@ func (u *userDatabase) InitOnce(ctx context.Context, users []*relation.UserModel
}
// Determine which users are missing from the database.
missingUsers := utils.SliceAnySub(users, existingUsers, func(e *relation.UserModel) string {
missingUsers := datautil.SliceAnySub(users, existingUsers, func(e *relation.UserModel) string {
return e.UserID
})
@@ -127,7 +125,7 @@ func (u *userDatabase) FindWithError(ctx context.Context, userIDs []string) (use
return
}
if len(users) != len(userIDs) {
err = errs.ErrRecordNotFound.Wrap("userID not found")
err = errs.ErrRecordNotFound.WrapMsg("userID not found")
}
return
}
@@ -137,12 +135,10 @@ func (u *userDatabase) Find(ctx context.Context, userIDs []string) (users []*rel
return u.cache.GetUsersInfo(ctx, userIDs)
}
// Find userInfo By Nickname.
func (u *userDatabase) FindByNickname(ctx context.Context, nickname string) (users []*relation.UserModel, err error) {
return u.userDB.TakeByNickname(ctx, nickname)
}
// Find notificationAccouts.
func (u *userDatabase) FindNotification(ctx context.Context, level int64) (users []*relation.UserModel, err error) {
return u.userDB.TakeNotification(ctx, level)
}
@@ -153,20 +149,12 @@ func (u *userDatabase) Create(ctx context.Context, users []*relation.UserModel)
if err = u.userDB.Create(ctx, users); err != nil {
return err
}
return u.cache.DelUsersInfo(utils.Slice(users, func(e *relation.UserModel) string {
return u.cache.DelUsersInfo(datautil.Slice(users, func(e *relation.UserModel) string {
return e.UserID
})...).ExecDel(ctx)
})
}
//// Update (non-zero value) externally guarantees that userID exists.
//func (u *userDatabase) Update(ctx context.Context, user *relation.UserModel) (err error) {
// if err := u.userDB.Update(ctx, user); err != nil {
// return err
// }
// return u.cache.DelUsersInfo(user.UserID).ExecDel(ctx)
//}
// UpdateByMap update (zero value) externally guarantees that userID exists.
func (u *userDatabase) UpdateByMap(ctx context.Context, userID string, args map[string]any) (err error) {
return u.tx.Transaction(ctx, func(ctx context.Context) error {
@@ -186,13 +174,7 @@ func (u *userDatabase) PageFindUser(ctx context.Context, level1 int64, level2 in
return u.userDB.PageFindUser(ctx, level1, level2, pagination)
}
func (u *userDatabase) PageFindUserWithKeyword(
ctx context.Context,
level1 int64,
level2 int64,
userID, nickName string,
pagination pagination.Pagination,
) (count int64, users []*relation.UserModel, err error) {
func (u *userDatabase) PageFindUserWithKeyword(ctx context.Context, level1 int64, level2 int64, userID, nickName string, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error) {
return u.userDB.PageFindUserWithKeyword(ctx, level1, level2, userID, nickName, pagination)
}
@@ -267,19 +249,24 @@ func (u *userDatabase) GetUserStatus(ctx context.Context, userIDs []string) ([]*
func (u *userDatabase) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error {
return u.cache.SetUserStatus(ctx, userID, status, platformID)
}
func (u *userDatabase) AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error {
return u.userDB.AddUserCommand(ctx, userID, Type, UUID, value, ex)
}
func (u *userDatabase) DeleteUserCommand(ctx context.Context, userID string, Type int32, UUID string) error {
return u.userDB.DeleteUserCommand(ctx, userID, Type, UUID)
}
func (u *userDatabase) UpdateUserCommand(ctx context.Context, userID string, Type int32, UUID string, val map[string]any) error {
return u.userDB.UpdateUserCommand(ctx, userID, Type, UUID, val)
}
func (u *userDatabase) GetUserCommands(ctx context.Context, userID string, Type int32) ([]*user.CommandInfoResp, error) {
commands, err := u.userDB.GetUserCommand(ctx, userID, Type)
return commands, err
}
func (u *userDatabase) GetAllUserCommands(ctx context.Context, userID string) ([]*user.AllCommandInfoResp, error) {
commands, err := u.userDB.GetAllUserCommand(ctx, userID)
return commands, err