IM/open-im-server
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into main

Browse Source
This commit is contained in:
skiffer-git
2024-04-28 11:56:58 +08:00
committed by GitHub
parent 381fd1beeb 74f4fdb156
commit 6a721f2eab
455 changed files with 12206 additions and 16474 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/common/db/cache/black.go
Vendored
+3 -3
View File
@@ -18,11 +18,11 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/log"
"github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/log"
"github.com/redis/go-redis/v9"
)
@@ -48,10 +48,10 @@ type BlackCacheRedis struct {
blackDB relationtb.BlackModelInterface
}
func NewBlackCacheRedis(rdb redis.UniversalClient, blackDB relationtb.BlackModelInterface, options rockscache.Options) BlackCache {
func NewBlackCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, blackDB relationtb.BlackModelInterface, options rockscache.Options) BlackCache {
rcClient := rockscache.NewClient(rdb, options)
mc := NewMetaCacheRedis(rcClient)
b := config.Config.LocalCache.Friend
b := localCache.Friend
log.ZDebug(context.Background(), "black local cache init", "Topic", b.Topic, "SlotNum", b.SlotNum, "SlotSize", b.SlotSize, "enable", b.Enable())
mc.SetTopic(b.Topic)
mc.SetRawRedisClient(rdb)
pkg/common/db/cache/config.go
Vendored
+12 -9
View File
@@ -27,29 +27,26 @@ var (
subscribe map[string][]string
)
func getPublishKey(topic string, key []string) []string {
if topic == "" || len(key) == 0 {
return nil
}
func InitLocalCache(localCache *config.LocalCache) {
once.Do(func() {
list := []struct {
Local config.LocalCache
Local config.CacheConfig
Keys []string
}{
{
Local: config.Config.LocalCache.User,
Local: localCache.User,
Keys: []string{cachekey.UserInfoKey, cachekey.UserGlobalRecvMsgOptKey},
},
{
Local: config.Config.LocalCache.Group,
Local: localCache.Group,
Keys: []string{cachekey.GroupMemberIDsKey, cachekey.GroupInfoKey, cachekey.GroupMemberInfoKey},
},
{
Local: config.Config.LocalCache.Friend,
Local: localCache.Friend,
Keys: []string{cachekey.FriendIDsKey, cachekey.BlackIDsKey},
},
{
Local: config.Config.LocalCache.Conversation,
Local: localCache.Conversation,
Keys: []string{cachekey.ConversationKey, cachekey.ConversationIDsKey, cachekey.ConversationNotReceiveMessageUserIDsKey},
},
}
@@ -60,6 +57,12 @@ func getPublishKey(topic string, key []string) []string {
}
}
})
}
func getPublishKey(topic string, key []string) []string {
if topic == "" || len(key) == 0 {
return nil
}
prefix, ok := subscribe[topic]
if !ok {
return nil
pkg/common/db/cache/conversation.go
Vendored
+29 -52
View File
@@ -20,24 +20,25 @@ import (
"strings"
"time"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/utils"
"github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/utils/datautil"
"github.com/openimsdk/tools/utils/encrypt"
"github.com/redis/go-redis/v9"
)
const (
//conversationKey = "CONVERSATION:"
//conversationIDsKey = "CONVERSATION_IDS:"
//conversationIDsHashKey = "CONVERSATION_IDS_HASH:"
//conversationHasReadSeqKey = "CONVERSATION_HAS_READ_SEQ:"
//recvMsgOptKey = "RECV_MSG_OPT:"
//superGroupRecvMsgNotNotifyUserIDsKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:"
//superGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:"
//conversationNotReceiveMessageUserIDsKey = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:".
// ConversationKey = "CONVERSATION:"
// conversationIDsKey = "CONVERSATION_IDS:"
// conversationIDsHashKey = "CONVERSATION_IDS_HASH:"
// conversationHasReadSeqKey = "CONVERSATION_HAS_READ_SEQ:"
// recvMsgOptKey = "RECV_MSG_OPT:"
// superGroupRecvMsgNotNotifyUserIDsKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:"
// superGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:"
// conversationNotReceiveMessageUserIDsKey = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:".
conversationExpireTime = time.Second * 60 * 60 * 12
)
@@ -66,13 +67,13 @@ type ConversationCache interface {
GetUserRecvMsgOpt(ctx context.Context, ownerUserID, conversationID string) (opt int, err error)
DelUserRecvMsgOpt(ownerUserID, conversationID string) ConversationCache
// get one super group recv msg but do not notification userID list
//GetSuperGroupRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) (userIDs []string, err error)
// GetSuperGroupRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) (userIDs []string, err error)
DelSuperGroupRecvMsgNotNotifyUserIDs(groupID string) ConversationCache
// get one super group recv msg but do not notification userID list hash
//GetSuperGroupRecvMsgNotNotifyUserIDsHash(ctx context.Context, groupID string) (hash uint64, err error)
// GetSuperGroupRecvMsgNotNotifyUserIDsHash(ctx context.Context, groupID string) (hash uint64, err error)
DelSuperGroupRecvMsgNotNotifyUserIDsHash(groupID string) ConversationCache
//GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error)
// GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error)
DelUserAllHasReadSeqs(ownerUserID string, conversationIDs ...string) ConversationCache
GetConversationsByConversationID(ctx context.Context,
@@ -82,10 +83,10 @@ type ConversationCache interface {
DelConversationNotReceiveMessageUserIDs(conversationIDs ...string) ConversationCache
}
func NewConversationRedis(rdb redis.UniversalClient, opts rockscache.Options, db relationtb.ConversationModelInterface) ConversationCache {
func NewConversationRedis(rdb redis.UniversalClient, localCache *config.LocalCache, opts rockscache.Options, db relationtb.ConversationModelInterface) ConversationCache {
rcClient := rockscache.NewClient(rdb, opts)
mc := NewMetaCacheRedis(rcClient)
c := config.Config.LocalCache.Conversation
c := localCache.Conversation
log.ZDebug(context.Background(), "black local cache init", "Topic", c.Topic, "SlotNum", c.SlotNum, "SlotSize", c.SlotSize, "enable", c.Enable())
mc.SetTopic(c.Topic)
mc.SetRawRedisClient(rdb)
@@ -104,11 +105,11 @@ type ConversationRedisCache struct {
expireTime time.Duration
}
//func NewNewConversationRedis(
// func NewNewConversationRedis(
// rdb redis.UniversalClient,
// conversationDB *relation.ConversationGorm,
// options rockscache.Options,
//) ConversationCache {
// ) ConversationCache {
// rcClient := rockscache.NewClient(rdb, options)
//
// return &ConversationRedisCache{
@@ -188,9 +189,9 @@ func (c *ConversationRedisCache) GetUserConversationIDsHash(ctx context.Context,
if err != nil {
return 0, err
}
utils.Sort(conversationIDs, true)
datautil.Sort(conversationIDs, true)
bi := big.NewInt(0)
bi.SetString(utils.Md5(strings.Join(conversationIDs, ";"))[0:8], 16)
bi.SetString(encrypt.Md5(strings.Join(conversationIDs, ";"))[0:8], 16)
return bi.Uint64(), nil
},
)
@@ -232,15 +233,15 @@ func (c *ConversationRedisCache) DelConversations(ownerUserID string, conversati
// }
// }
// return 0, errors.New("not found key:" + key + " in keys")
// return 0, errs.New("not found key:" + key + " in keys")
// }
func (c *ConversationRedisCache) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*relationtb.ConversationModel, error) {
//var keys []string
//for _, conversarionID := range conversationIDs {
// var keys []string
// for _, conversarionID := range conversationIDs {
// keys = append(keys, c.getConversationKey(ownerUserID, conversarionID))
//}
//return batchGetCache(
// return batchGetCache(
// ctx,
// c.rcClient,
// keys,
@@ -262,11 +263,11 @@ func (c *ConversationRedisCache) GetUserAllConversations(ctx context.Context, ow
if err != nil {
return nil, err
}
//var keys []string
//for _, conversarionID := range conversationIDs {
// var keys []string
// for _, conversarionID := range conversationIDs {
// keys = append(keys, c.getConversationKey(ownerUserID, conversarionID))
//}
//return batchGetCache(
// return batchGetCache(
// ctx,
// c.rcClient,
// keys,
@@ -285,7 +286,7 @@ func (c *ConversationRedisCache) GetUserRecvMsgOpt(ctx context.Context, ownerUse
})
}
//func (c *ConversationRedisCache) GetSuperGroupRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) (userIDs []string, err error) {
// func (c *ConversationRedisCache) GetSuperGroupRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) (userIDs []string, err error) {
// return getCache(ctx, c.rcClient, c.getSuperGroupRecvNotNotifyUserIDsKey(groupID), c.expireTime, func(ctx context.Context) (userIDs []string, err error) {
// return c.conversationDB.FindSuperGroupRecvMsgNotNotifyUserIDs(ctx, groupID)
// })
@@ -316,7 +317,7 @@ func (c *ConversationRedisCache) DelSuperGroupRecvMsgNotNotifyUserIDs(groupID st
return cache
}
//func (c *ConversationRedisCache) GetSuperGroupRecvMsgNotNotifyUserIDsHash(ctx context.Context, groupID string) (hash uint64, err error) {
// func (c *ConversationRedisCache) GetSuperGroupRecvMsgNotNotifyUserIDsHash(ctx context.Context, groupID string) (hash uint64, err error) {
// return getCache(ctx, c.rcClient, c.getSuperGroupRecvNotNotifyUserIDsHashKey(groupID), c.expireTime, func(ctx context.Context) (hash uint64, err error) {
// userIDs, err := c.GetSuperGroupRecvMsgNotNotifyUserIDs(ctx, groupID)
// if err != nil {
@@ -337,30 +338,6 @@ func (c *ConversationRedisCache) DelSuperGroupRecvMsgNotNotifyUserIDsHash(groupI
return cache
}
/* func (c *ConversationRedisCache) getUserAllHasReadSeqsIndex(conversationID string, conversationIDs []string) (int, error) {
for _i, _conversationID := range conversationIDs {
if _conversationID == conversationID {
return _i, nil
}
}
return 0, errors.New("not found key:" + conversationID + " in keys")
} */
/* func (c *ConversationRedisCache) GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error) {
conversationIDs, err := c.GetUserConversationIDs(ctx, ownerUserID)
if err != nil {
return nil, err
}
var keys []string
for _, conversarionID := range conversationIDs {
keys = append(keys, c.getConversationHasReadSeqKey(ownerUserID, conversarionID))
}
return batchGetCacheMap(ctx, c.rcClient, keys, conversationIDs, c.expireTime, c.getUserAllHasReadSeqsIndex, func(ctx context.Context) (map[string]int64, error) {
return c.conversationDB.GetUserAllHasReadSeqs(ctx, ownerUserID)
})
} */
func (c *ConversationRedisCache) DelUserAllHasReadSeqs(ownerUserID string, conversationIDs ...string) ConversationCache {
cache := c.NewCache()
for _, conversationID := range conversationIDs {
pkg/common/db/cache/doc.go
Vendored
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright © 2023 OpenIM. All rights reserved.
// Copyright © 2024 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
pkg/common/db/cache/friend.go
Vendored
+8 -8
View File
@@ -18,20 +18,20 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/utils"
"github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/utils/datautil"
"github.com/redis/go-redis/v9"
)
const (
friendExpireTime = time.Second * 60 * 60 * 12
//friendIDsKey = "FRIEND_IDS:"
//TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:"
//friendKey = "FRIEND_INFO:".
// FriendIDsKey = "FRIEND_IDS:"
// TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:"
// friendKey = "FRIEND_INFO:".
)
// FriendCache is an interface for caching friend-related data.
@@ -58,11 +58,11 @@ type FriendCacheRedis struct {
}
// NewFriendCacheRedis creates a new instance of FriendCacheRedis.
func NewFriendCacheRedis(rdb redis.UniversalClient, friendDB relationtb.FriendModelInterface,
func NewFriendCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, friendDB relationtb.FriendModelInterface,
options rockscache.Options) FriendCache {
rcClient := rockscache.NewClient(rdb, options)
mc := NewMetaCacheRedis(rcClient)
f := config.Config.LocalCache.Friend
f := localCache.Friend
log.ZDebug(context.Background(), "friend local cache init", "Topic", f.Topic, "SlotNum", f.SlotNum, "SlotSize", f.SlotSize, "enable", f.Enable())
mc.SetTopic(f.Topic)
mc.SetRawRedisClient(rdb)
@@ -129,7 +129,7 @@ func (f *FriendCacheRedis) GetTwoWayFriendIDs(ctx context.Context, ownerUserID s
if err != nil {
return nil, err
}
if utils.IsContain(ownerUserID, friendFriendID) {
if datautil.Contain(ownerUserID, friendFriendID...) {
twoWayFriendIDs = append(twoWayFriendIDs, ownerUserID)
}
}
pkg/common/db/cache/group.go
Vendored
+12 -18
View File
@@ -19,26 +19,19 @@ import (
"fmt"
"time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/utils"
"github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/utils/datautil"
"github.com/redis/go-redis/v9"
)
const (
groupExpireTime = time.Second * 60 * 60 * 12
//groupInfoKey = "GROUP_INFO:"
//groupMemberIDsKey = "GROUP_MEMBER_IDS:"
//groupMembersHashKey = "GROUP_MEMBERS_HASH2:"
//groupMemberInfoKey = "GROUP_MEMBER_INFO:"
//joinedGroupsKey = "JOIN_GROUPS_KEY:"
//groupMemberNumKey = "GROUP_MEMBER_NUM_CACHE:"
//groupRoleLevelMemberIDsKey = "GROUP_ROLE_LEVEL_MEMBER_IDS:".
)
type GroupHash interface {
@@ -94,6 +87,7 @@ type GroupCacheRedis struct {
func NewGroupCacheRedis(
rdb redis.UniversalClient,
localCache *config.LocalCache,
groupDB relationtb.GroupModelInterface,
groupMemberDB relationtb.GroupMemberModelInterface,
groupRequestDB relationtb.GroupRequestModelInterface,
@@ -102,7 +96,7 @@ func NewGroupCacheRedis(
) GroupCache {
rcClient := rockscache.NewClient(rdb, opts)
mc := NewMetaCacheRedis(rcClient)
g := config.Config.LocalCache.Group
g := localCache.Group
mc.SetTopic(g.Topic)
log.ZDebug(context.Background(), "group local cache init", "Topic", g.Topic, "SlotNum", g.SlotNum, "SlotSize", g.SlotSize, "enable", g.Enable())
mc.SetRawRedisClient(rdb)
@@ -227,7 +221,7 @@ func (g *GroupCacheRedis) DelGroupAllRoleLevel(groupID string) GroupCache {
func (g *GroupCacheRedis) GetGroupMembersHash(ctx context.Context, groupID string) (hashCode uint64, err error) {
if g.groupHash == nil {
return 0, errs.ErrInternalServer.Wrap("group hash is nil")
return 0, errs.ErrInternalServer.WrapMsg("group hash is nil")
}
return getCache(ctx, g.rcClient, g.getGroupMembersHashKey(groupID), g.expireTime, func(ctx context.Context) (uint64, error) {
return g.groupHash.GetGroupHash(ctx, groupID)
@@ -236,7 +230,7 @@ func (g *GroupCacheRedis) GetGroupMembersHash(ctx context.Context, groupID strin
func (g *GroupCacheRedis) GetGroupMemberHashMap(ctx context.Context, groupIDs []string) (map[string]*relationtb.GroupSimpleUserID, error) {
if g.groupHash == nil {
return nil, errs.ErrInternalServer.Wrap("group hash is nil")
return nil, errs.ErrInternalServer.WrapMsg("group hash is nil")
}
res := make(map[string]*relationtb.GroupSimpleUserID)
for _, groupID := range groupIDs {
@@ -244,7 +238,7 @@ func (g *GroupCacheRedis) GetGroupMemberHashMap(ctx context.Context, groupIDs []
if err != nil {
return nil, err
}
log.ZInfo(ctx, "GetGroupMemberHashMap", "groupID", groupID, "hash", hash)
log.ZDebug(ctx, "GetGroupMemberHashMap", "groupID", groupID, "hash", hash)
num, err := g.GetGroupMemberNum(ctx, groupID)
if err != nil {
return nil, err
@@ -330,11 +324,11 @@ func (g *GroupCacheRedis) GetGroupMembersPage(
return 0, nil, err
}
if userIDs != nil {
userIDs = utils.BothExist(userIDs, groupMemberIDs)
userIDs = datautil.BothExist(userIDs, groupMemberIDs)
} else {
userIDs = groupMemberIDs
}
groupMembers, err = g.GetGroupMembersInfo(ctx, groupID, utils.Paginate(userIDs, int(showNumber), int(showNumber)))
groupMembers, err = g.GetGroupMembersInfo(ctx, groupID, datautil.Paginate(userIDs, int(showNumber), int(showNumber)))
return uint32(len(userIDs)), groupMembers, err
}
@@ -390,7 +384,7 @@ func (g *GroupCacheRedis) GetGroupOwner(ctx context.Context, groupID string) (*r
return nil, err
}
if len(members) == 0 {
return nil, errs.ErrRecordNotFound.Wrap(fmt.Sprintf("group %s owner not found", groupID))
return nil, errs.ErrRecordNotFound.WrapMsg(fmt.Sprintf("group %s owner not found", groupID))
}
return members[0], nil
}
pkg/common/db/cache/init_redis.go
Vendored
-107
View File
@@ -1,107 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cache
import (
"context"
"errors"
"fmt"
"os"
"strings"
"time"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/mw/specialerror"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/redis/go-redis/v9"
)
var (
// singleton pattern.
redisClient redis.UniversalClient
)
const (
maxRetry = 10 // number of retries
)
// NewRedis Initialize redis connection.
func NewRedis(config *config.GlobalConfig) (redis.UniversalClient, error) {
if redisClient != nil {
return redisClient, nil
}
// Read configuration from environment variables
overrideConfigFromEnv(config)
if len(config.Redis.Address) == 0 {
return nil, errs.Wrap(errors.New("redis address is empty"))
}
specialerror.AddReplace(redis.Nil, errs.ErrRecordNotFound)
var rdb redis.UniversalClient
if len(config.Redis.Address) > 1 || config.Redis.ClusterMode {
rdb = redis.NewClusterClient(&redis.ClusterOptions{
Addrs: config.Redis.Address,
Username: config.Redis.Username,
Password: config.Redis.Password, // no password set
PoolSize: 50,
MaxRetries: maxRetry,
})
} else {
rdb = redis.NewClient(&redis.Options{
Addr: config.Redis.Address[0],
Username: config.Redis.Username,
Password: config.Redis.Password,
DB: 0, // use default DB
PoolSize: 100, // connection pool size
MaxRetries: maxRetry,
})
}
var err error
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
err = rdb.Ping(ctx).Err()
if err != nil {
errMsg := fmt.Sprintf("address:%s, username:%s, password:%s, clusterMode:%t, enablePipeline:%t", config.Redis.Address, config.Redis.Username,
config.Redis.Password, config.Redis.ClusterMode, config.Redis.EnablePipeline)
return nil, errs.Wrap(err, errMsg)
}
redisClient = rdb
return rdb, err
}
// overrideConfigFromEnv overrides configuration fields with environment variables if present.
func overrideConfigFromEnv(config *config.GlobalConfig) {
if envAddr := os.Getenv("REDIS_ADDRESS"); envAddr != "" {
if envPort := os.Getenv("REDIS_PORT"); envPort != "" {
addresses := strings.Split(envAddr, ",")
for i, addr := range addresses {
addresses[i] = addr + ":" + envPort
}
config.Redis.Address = addresses
} else {
config.Redis.Address = strings.Split(envAddr, ",")
}
}
if envUser := os.Getenv("REDIS_USERNAME"); envUser != "" {
config.Redis.Username = envUser
}
if envPass := os.Getenv("REDIS_PASSWORD"); envPass != "" {
config.Redis.Password = envPass
}
}
pkg/common/db/cache/meta_cache.go
Vendored
+12 -13
View File
@@ -17,15 +17,14 @@ package cache
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/mw/specialerror"
"github.com/OpenIMSDK/tools/utils"
"github.com/dtm-labs/rockscache"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/mw/specialerror"
"github.com/openimsdk/tools/utils/datautil"
"github.com/redis/go-redis/v9"
)
@@ -35,7 +34,7 @@ const (
retryInterval = time.Millisecond * 100
)
var errIndex = errors.New("err index")
var errIndex = errs.New("err index")
type metaCache interface {
ExecDel(ctx context.Context, distinct ...bool) error
@@ -74,7 +73,7 @@ func (m *metaCacheRedis) Copy() metaCache {
keys: keys,
maxRetryTimes: m.maxRetryTimes,
retryInterval: m.retryInterval,
redisClient: redisClient,
redisClient: m.redisClient,
}
}
@@ -88,7 +87,7 @@ func (m *metaCacheRedis) SetRawRedisClient(cli redis.UniversalClient) {
func (m *metaCacheRedis) ExecDel(ctx context.Context, distinct ...bool) error {
if len(distinct) > 0 && distinct[0] {
m.keys = utils.Distinct(m.keys)
m.keys = datautil.Distinct(m.keys)
}
if len(m.keys) > 0 {
log.ZDebug(ctx, "delete cache", "topic", m.topic, "keys", m.keys)
@@ -150,7 +149,7 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin
}
bs, err := json.Marshal(t)
if err != nil {
return "", errs.Wrap(err, "marshal failed")
return "", errs.WrapMsg(err, "marshal failed")
}
write = true
@@ -163,12 +162,12 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin
return t, nil
}
if v == "" {
return t, errs.ErrRecordNotFound.Wrap("cache is not found")
return t, errs.ErrRecordNotFound.WrapMsg("cache is not found")
}
err = json.Unmarshal([]byte(v), &t)
if err != nil {
errInfo := fmt.Sprintf("cache json.Unmarshal failed, key:%s, value:%s, expire:%s", key, v, expire)
return t, errs.Wrap(err, errInfo)
return t, errs.WrapMsg(err, errInfo)
}
return t, nil
@@ -240,14 +239,14 @@ func batchGetCache2[T any, K comparable](
return res, nil
}
//func batchGetCacheMap[T any](
// func batchGetCacheMap[T any](
// ctx context.Context,
// rcClient *rockscache.Client,
// keys, originKeys []string,
// expire time.Duration,
// keyIndexFn func(s string, keys []string) (int, error),
// fn func(ctx context.Context) (map[string]T, error),
//) (map[string]T, error) {
// ) (map[string]T, error) {
// batchMap, err := rcClient.FetchBatch2(ctx, keys, expire, func(idxs []int) (m map[int]string, err error) {
// tArrays, err := fn(ctx)
// if err != nil {
pkg/common/db/cache/msg.go
Vendored
+377 -644
View File
File diff suppressed because it is too large Load Diff
pkg/common/db/cache/msg_test.go
Vendored
+1 -1
View File
@@ -20,7 +20,7 @@ import (
"math/rand"
"testing"
"github.com/OpenIMSDK/protocol/sdkws"
"github.com/openimsdk/protocol/sdkws"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/assert"
)
pkg/common/db/cache/s3.go
Vendored
+17 -15
View File
@@ -16,12 +16,14 @@ package cache
import (
"context"
"github.com/openimsdk/tools/s3/cont"
"github.com/openimsdk/tools/s3/minio"
"strconv"
"time"
"github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/s3"
"github.com/redis/go-redis/v9"
)
@@ -83,7 +85,7 @@ type S3Cache interface {
DelS3Key(engine string, keys ...string) S3Cache
}
func NewS3Cache(rdb redis.UniversalClient, s3 s3.Interface) S3Cache {
func NewS3Cache(rdb redis.UniversalClient, s3 s3.Interface) cont.S3Cache {
rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions())
return &s3CacheRedis{
rcClient: rcClient,
@@ -100,7 +102,7 @@ type s3CacheRedis struct {
expireTime time.Duration
}
func (g *s3CacheRedis) NewCache() S3Cache {
func (g *s3CacheRedis) newCache() *s3CacheRedis {
return &s3CacheRedis{
rcClient: g.rcClient,
expireTime: g.expireTime,
@@ -109,14 +111,14 @@ func (g *s3CacheRedis) NewCache() S3Cache {
}
}
func (g *s3CacheRedis) DelS3Key(engine string, keys ...string) S3Cache {
s3cache := g.NewCache()
func (g *s3CacheRedis) DelS3Key(ctx context.Context, engine string, keys ...string) error {
s3cache := g.newCache()
ks := make([]string, 0, len(keys))
for _, key := range keys {
ks = append(ks, g.getS3Key(engine, key))
}
s3cache.AddKeys(ks...)
return s3cache
return s3cache.ExecDel(ctx)
}
func (g *s3CacheRedis) getS3Key(engine string, name string) string {
@@ -137,7 +139,7 @@ type MinioCache interface {
DelImageThumbnailKey(key string, format string, width int, height int) MinioCache
}
func NewMinioCache(rdb redis.UniversalClient) MinioCache {
func NewMinioCache(rdb redis.UniversalClient) minio.Cache {
rcClient := rockscache.NewClient(rdb, rockscache.NewDefaultOptions())
return &minioCacheRedis{
rcClient: rcClient,
@@ -152,7 +154,7 @@ type minioCacheRedis struct {
expireTime time.Duration
}
func (g *minioCacheRedis) NewCache() MinioCache {
func (g *minioCacheRedis) newCache() *minioCacheRedis {
return &minioCacheRedis{
rcClient: g.rcClient,
expireTime: g.expireTime,
@@ -160,20 +162,20 @@ func (g *minioCacheRedis) NewCache() MinioCache {
}
}
func (g *minioCacheRedis) DelObjectImageInfoKey(keys ...string) MinioCache {
s3cache := g.NewCache()
func (g *minioCacheRedis) DelObjectImageInfoKey(ctx context.Context, keys ...string) error {
s3cache := g.newCache()
ks := make([]string, 0, len(keys))
for _, key := range keys {
ks = append(ks, g.getObjectImageInfoKey(key))
}
s3cache.AddKeys(ks...)
return s3cache
return s3cache.ExecDel(ctx)
}
func (g *minioCacheRedis) DelImageThumbnailKey(key string, format string, width int, height int) MinioCache {
s3cache := g.NewCache()
func (g *minioCacheRedis) DelImageThumbnailKey(ctx context.Context, key string, format string, width int, height int) error {
s3cache := g.newCache()
s3cache.AddKeys(g.getMinioImageThumbnailKey(key, format, width, height))
return s3cache
return s3cache.ExecDel(ctx)
}
func (g *minioCacheRedis) getObjectImageInfoKey(key string) string {
@@ -184,7 +186,7 @@ func (g *minioCacheRedis) getMinioImageThumbnailKey(key string, format string, w
return "MINIO:THUMBNAIL:" + format + ":w" + strconv.Itoa(width) + ":h" + strconv.Itoa(height) + ":" + key
}
func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*MinioImageInfo, error)) (*MinioImageInfo, error) {
func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*minio.ImageInfo, error)) (*minio.ImageInfo, error) {
info, err := getCache(ctx, g.rcClient, g.getObjectImageInfoKey(key), g.expireTime, fn)
if err != nil {
return nil, err
pkg/common/db/cache/seq.go
Vendored
+182
View File
@@ -0,0 +1,182 @@
package cache
import (
"context"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/utils/stringutil"
"github.com/redis/go-redis/v9"
)
type SeqCache interface {
SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error
GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error)
GetMaxSeq(ctx context.Context, conversationID string) (int64, error)
SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error
SetMinSeqs(ctx context.Context, seqs map[string]int64) error
GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error)
GetMinSeq(ctx context.Context, conversationID string) (int64, error)
GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error)
GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error)
SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error
// seqs map: key userID value minSeq
SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error)
// seqs map: key conversationID value minSeq
SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error
// has read seq
SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error
// k: user, v: seq
SetHasReadSeqs(ctx context.Context, conversationID string, hasReadSeqs map[string]int64) error
// k: conversation, v :seq
UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error
GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error)
GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error)
}
func NewSeqCache(rdb redis.UniversalClient) SeqCache {
return &seqCache{rdb: rdb}
}
type seqCache struct {
rdb redis.UniversalClient
}
func (c *seqCache) getMaxSeqKey(conversationID string) string {
return maxSeq + conversationID
}
func (c *seqCache) getMinSeqKey(conversationID string) string {
return minSeq + conversationID
}
func (c *seqCache) getHasReadSeqKey(conversationID string, userID string) string {
return hasReadSeq + userID + ":" + conversationID
}
func (c *seqCache) getConversationUserMinSeqKey(conversationID, userID string) string {
return conversationUserMinSeq + conversationID + "u:" + userID
}
func (c *seqCache) setSeq(ctx context.Context, conversationID string, seq int64, getkey func(conversationID string) string) error {
return errs.Wrap(c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err())
}
func (c *seqCache) getSeq(ctx context.Context, conversationID string, getkey func(conversationID string) string) (int64, error) {
val, err := c.rdb.Get(ctx, getkey(conversationID)).Int64()
if err != nil {
return 0, errs.Wrap(err)
}
return val, nil
}
func (c *seqCache) getSeqs(ctx context.Context, items []string, getkey func(s string) string) (m map[string]int64, err error) {
m = make(map[string]int64, len(items))
for i, v := range items {
res, err := c.rdb.Get(ctx, getkey(v)).Result()
if err != nil && err != redis.Nil {
return nil, errs.Wrap(err)
}
val := stringutil.StringToInt64(res)
if val != 0 {
m[items[i]] = val
}
}
return m, nil
}
func (c *seqCache) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error {
return c.setSeq(ctx, conversationID, maxSeq, c.getMaxSeqKey)
}
func (c *seqCache) GetMaxSeqs(ctx context.Context, conversationIDs []string) (m map[string]int64, err error) {
return c.getSeqs(ctx, conversationIDs, c.getMaxSeqKey)
}
func (c *seqCache) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) {
return c.getSeq(ctx, conversationID, c.getMaxSeqKey)
}
func (c *seqCache) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error {
return c.setSeq(ctx, conversationID, minSeq, c.getMinSeqKey)
}
func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey func(key string) string) error {
for conversationID, seq := range seqs {
if err := c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err(); err != nil {
return errs.Wrap(err)
}
}
return nil
}
func (c *seqCache) SetMinSeqs(ctx context.Context, seqs map[string]int64) error {
return c.setSeqs(ctx, seqs, c.getMinSeqKey)
}
func (c *seqCache) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) {
return c.getSeqs(ctx, conversationIDs, c.getMinSeqKey)
}
func (c *seqCache) GetMinSeq(ctx context.Context, conversationID string) (int64, error) {
return c.getSeq(ctx, conversationID, c.getMinSeqKey)
}
func (c *seqCache) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
val, err := c.rdb.Get(ctx, c.getConversationUserMinSeqKey(conversationID, userID)).Int64()
if err != nil {
return 0, errs.Wrap(err)
}
return val, nil
}
func (c *seqCache) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (m map[string]int64, err error) {
return c.getSeqs(ctx, userIDs, func(userID string) string {
return c.getConversationUserMinSeqKey(conversationID, userID)
})
}
func (c *seqCache) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error {
return errs.Wrap(c.rdb.Set(ctx, c.getConversationUserMinSeqKey(conversationID, userID), minSeq, 0).Err())
}
func (c *seqCache) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) {
return c.setSeqs(ctx, seqs, func(userID string) string {
return c.getConversationUserMinSeqKey(conversationID, userID)
})
}
func (c *seqCache) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) {
return c.setSeqs(ctx, seqs, func(conversationID string) string {
return c.getConversationUserMinSeqKey(conversationID, userID)
})
}
func (c *seqCache) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error {
return errs.Wrap(c.rdb.Set(ctx, c.getHasReadSeqKey(conversationID, userID), hasReadSeq, 0).Err())
}
func (c *seqCache) SetHasReadSeqs(ctx context.Context, conversationID string, hasReadSeqs map[string]int64) error {
return c.setSeqs(ctx, hasReadSeqs, func(userID string) string {
return c.getHasReadSeqKey(conversationID, userID)
})
}
func (c *seqCache) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error {
return c.setSeqs(ctx, hasReadSeqs, func(conversationID string) string {
return c.getHasReadSeqKey(conversationID, userID)
})
}
func (c *seqCache) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) {
return c.getSeqs(ctx, conversationIDs, func(conversationID string) string {
return c.getHasReadSeqKey(conversationID, userID)
})
}
func (c *seqCache) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) {
val, err := c.rdb.Get(ctx, c.getHasReadSeqKey(conversationID, userID)).Int64()
if err != nil {
return 0, err
}
return val, nil
}
pkg/common/db/cache/third.go
Vendored
+85
View File
@@ -0,0 +1,85 @@
package cache
import (
"context"
"github.com/openimsdk/tools/errs"
"github.com/redis/go-redis/v9"
"strconv"
"time"
)
type ThirdCache interface {
SetFcmToken(ctx context.Context, account string, platformID int, fcmToken string, expireTime int64) (err error)
GetFcmToken(ctx context.Context, account string, platformID int) (string, error)
DelFcmToken(ctx context.Context, account string, platformID int) error
IncrUserBadgeUnreadCountSum(ctx context.Context, userID string) (int, error)
SetUserBadgeUnreadCountSum(ctx context.Context, userID string, value int) error
GetUserBadgeUnreadCountSum(ctx context.Context, userID string) (int, error)
SetGetuiToken(ctx context.Context, token string, expireTime int64) error
GetGetuiToken(ctx context.Context) (string, error)
SetGetuiTaskID(ctx context.Context, taskID string, expireTime int64) error
GetGetuiTaskID(ctx context.Context) (string, error)
}
func NewThirdCache(rdb redis.UniversalClient) ThirdCache {
return &thirdCache{rdb: rdb}
}
type thirdCache struct {
rdb redis.UniversalClient
}
func (c *thirdCache) SetFcmToken(ctx context.Context, account string, platformID int, fcmToken string, expireTime int64) (err error) {
return errs.Wrap(c.rdb.Set(ctx, FCM_TOKEN+account+":"+strconv.Itoa(platformID), fcmToken, time.Duration(expireTime)*time.Second).Err())
}
func (c *thirdCache) GetFcmToken(ctx context.Context, account string, platformID int) (string, error) {
val, err := c.rdb.Get(ctx, FCM_TOKEN+account+":"+strconv.Itoa(platformID)).Result()
if err != nil {
return "", errs.Wrap(err)
}
return val, nil
}
func (c *thirdCache) DelFcmToken(ctx context.Context, account string, platformID int) error {
return errs.Wrap(c.rdb.Del(ctx, FCM_TOKEN+account+":"+strconv.Itoa(platformID)).Err())
}
func (c *thirdCache) IncrUserBadgeUnreadCountSum(ctx context.Context, userID string) (int, error) {
seq, err := c.rdb.Incr(ctx, userBadgeUnreadCountSum+userID).Result()
return int(seq), errs.Wrap(err)
}
func (c *thirdCache) SetUserBadgeUnreadCountSum(ctx context.Context, userID string, value int) error {
return errs.Wrap(c.rdb.Set(ctx, userBadgeUnreadCountSum+userID, value, 0).Err())
}
func (c *thirdCache) GetUserBadgeUnreadCountSum(ctx context.Context, userID string) (int, error) {
val, err := c.rdb.Get(ctx, userBadgeUnreadCountSum+userID).Int()
return val, errs.Wrap(err)
}
func (c *thirdCache) SetGetuiToken(ctx context.Context, token string, expireTime int64) error {
return errs.Wrap(c.rdb.Set(ctx, getuiToken, token, time.Duration(expireTime)*time.Second).Err())
}
func (c *thirdCache) GetGetuiToken(ctx context.Context) (string, error) {
val, err := c.rdb.Get(ctx, getuiToken).Result()
if err != nil {
return "", errs.Wrap(err)
}
return val, nil
}
func (c *thirdCache) SetGetuiTaskID(ctx context.Context, taskID string, expireTime int64) error {
return errs.Wrap(c.rdb.Set(ctx, getuiTaskID, taskID, time.Duration(expireTime)*time.Second).Err())
}
func (c *thirdCache) GetGetuiTaskID(ctx context.Context) (string, error) {
val, err := c.rdb.Get(ctx, getuiTaskID).Result()
if err != nil {
return "", errs.Wrap(err)
}
return val, nil
}
pkg/common/db/cache/token.go
Vendored
+56
View File
@@ -0,0 +1,56 @@
package cache
import (
"context"
"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/utils/stringutil"
"github.com/redis/go-redis/v9"
)
func NewTokenCacheModel(rdb redis.UniversalClient) TokenModel {
return &tokenCache{
rdb: rdb,
}
}
type TokenModel interface {
AddTokenFlag(ctx context.Context, userID string, platformID int, token string, flag int) error
GetTokensWithoutError(ctx context.Context, userID string, platformID int) (map[string]int, error)
SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error
DeleteTokenByUidPid(ctx context.Context, userID string, platformID int, fields []string) error
}
type tokenCache struct {
rdb redis.UniversalClient
}
func (c *tokenCache) AddTokenFlag(ctx context.Context, userID string, platformID int, token string, flag int) error {
return errs.Wrap(c.rdb.HSet(ctx, cachekey.GetTokenKey(userID, platformID), token, flag).Err())
}
func (c *tokenCache) GetTokensWithoutError(ctx context.Context, userID string, platformID int) (map[string]int, error) {
m, err := c.rdb.HGetAll(ctx, cachekey.GetTokenKey(userID, platformID)).Result()
if err != nil {
return nil, errs.Wrap(err)
}
mm := make(map[string]int)
for k, v := range m {
mm[k] = stringutil.StringToInt(v)
}
return mm, nil
}
func (c *tokenCache) SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error {
mm := make(map[string]any)
for k, v := range m {
mm[k] = v
}
return errs.Wrap(c.rdb.HSet(ctx, cachekey.GetTokenKey(userID, platformID), mm).Err())
}
func (c *tokenCache) DeleteTokenByUidPid(ctx context.Context, userID string, platformID int, fields []string) error {
return errs.Wrap(c.rdb.HDel(ctx, cachekey.GetTokenKey(userID, platformID), fields...).Err())
}
pkg/common/db/cache/user.go
Vendored
+9 -16
View File
@@ -22,25 +22,22 @@ import (
"strconv"
"time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/protocol/user"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/protocol/user"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/log"
"github.com/redis/go-redis/v9"
)
const (
userExpireTime = time.Second * 60 * 60 * 12
//userInfoKey = "USER_INFO:".
userGlobalRecvMsgOptKey = "USER_GLOBAL_RECV_MSG_OPT_KEY:"
userExpireTime = time.Second * 60 * 60 * 12
olineStatusKey = "ONLINE_STATUS:"
userOlineStatusExpireTime = time.Second * 60 * 60 * 24
statusMod = 501
platformID = "_PlatformIDSuffix"
)
type UserCache interface {
@@ -58,20 +55,16 @@ type UserCache interface {
type UserCacheRedis struct {
metaCache
rdb redis.UniversalClient
//userDB relationtb.UserModelInterface
// userDB relationtb.UserModelInterface
userDB relationtb.UserModelInterface
expireTime time.Duration
rcClient *rockscache.Client
}
func NewUserCacheRedis(
rdb redis.UniversalClient,
userDB relationtb.UserModelInterface,
options rockscache.Options,
) UserCache {
func NewUserCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, userDB relationtb.UserModelInterface, options rockscache.Options) UserCache {
rcClient := rockscache.NewClient(rdb, options)
mc := NewMetaCacheRedis(rcClient)
u := config.Config.LocalCache.User
u := localCache.User
log.ZDebug(context.Background(), "user local cache init", "Topic", u.Topic, "SlotNum", u.SlotNum, "SlotSize", u.SlotSize, "enable", u.Enable())
mc.SetTopic(u.Topic)
mc.SetRawRedisClient(rdb)
@@ -294,7 +287,7 @@ func (u *UserCacheRedis) refreshStatusOnline(ctx context.Context, userID string,
onlineStatus.UserID = userID
newjsonData, err := json.Marshal(&onlineStatus)
if err != nil {
return errs.Wrap(err, "json.Marshal failed")
return errs.WrapMsg(err, "json.Marshal failed")
}
_, err = u.rdb.HSet(ctx, key, userID, string(newjsonData)).Result()
if err != nil {
pkg/common/db/controller/auth.go
+14 -10
View File
@@ -17,13 +17,12 @@ package controller
import (
"context"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/tokenverify"
"github.com/golang-jwt/jwt/v4"
"github.com/openimsdk/open-im-server/v3/pkg/authverify"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/tokenverify"
)
type AuthDatabase interface {
@@ -31,17 +30,18 @@ type AuthDatabase interface {
GetTokensWithoutError(ctx context.Context, userID string, platformID int) (map[string]int, error)
// Create token
CreateToken(ctx context.Context, userID string, platformID int) (string, error)
SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error
}
type authDatabase struct {
cache cache.MsgModel
cache cache.TokenModel
accessSecret string
accessExpire int64
config *config.GlobalConfig
}
func NewAuthDatabase(cache cache.MsgModel, accessSecret string, accessExpire int64, config *config.GlobalConfig) AuthDatabase {
return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire, config: config}
func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire int64) AuthDatabase {
return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire}
}
// If the result is empty.
@@ -49,6 +49,10 @@ func (a *authDatabase) GetTokensWithoutError(ctx context.Context, userID string,
return a.cache.GetTokensWithoutError(ctx, userID, platformID)
}
func (a *authDatabase) SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error {
return a.cache.SetTokenMapByUidPid(ctx, userID, platformID, m)
}
// Create Token.
func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformID int) (string, error) {
tokens, err := a.cache.GetTokensWithoutError(ctx, userID, platformID)
@@ -57,7 +61,7 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI
}
var deleteTokenKey []string
for k, v := range tokens {
_, err = tokenverify.GetClaimFromToken(k, authverify.Secret(a.config.Secret))
_, err = tokenverify.GetClaimFromToken(k, authverify.Secret(a.accessSecret))
if err != nil || v != constant.NormalToken {
deleteTokenKey = append(deleteTokenKey, k)
}
@@ -73,7 +77,7 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
tokenString, err := token.SignedString([]byte(a.accessSecret))
if err != nil {
return "", errs.Wrap(err, "token.SignedString")
return "", errs.WrapMsg(err, "token.SignedString")
}
return tokenString, a.cache.AddTokenFlag(ctx, userID, platformID, tokenString, constant.NormalToken)
}
pkg/common/db/controller/black.go
+4 -4
View File
@@ -17,11 +17,11 @@ package controller
import (
"context"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/utils/datautil"
)
type BlackDatabase interface {
@@ -86,7 +86,7 @@ func (b *blackDatabase) CheckIn(ctx context.Context, userID1, userID2 string) (i
return
}
log.ZDebug(ctx, "blackIDs", "user1BlackIDs", userID1BlackIDs, "user2BlackIDs", userID2BlackIDs)
return utils.IsContain(userID2, userID1BlackIDs), utils.IsContain(userID1, userID2BlackIDs), nil
return datautil.Contain(userID2, userID1BlackIDs...), datautil.Contain(userID1, userID2BlackIDs...), nil
}
// FindBlackIDs Get Blacklist List.
pkg/common/db/controller/conversation.go
+19 -18
View File
@@ -18,14 +18,15 @@ import (
"context"
"time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/db/tx"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/utils/datautil"
"github.com/openimsdk/tools/utils/stringutil"
)
type ConversationDatabase interface {
@@ -62,11 +63,11 @@ type ConversationDatabase interface {
GetConversationIDsNeedDestruct(ctx context.Context) ([]*relationtb.ConversationModel, error)
// GetConversationNotReceiveMessageUserIDs gets user IDs for users in a conversation who have not received messages.
GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error)
//GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error)
//FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error)
// GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error)
// FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error)
}
func NewConversationDatabase(conversation relationtb.ConversationModelInterface, cache cache.ConversationCache, tx tx.CtxTx) ConversationDatabase {
func NewConversationDatabase(conversation relationtb.ConversationModelInterface, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase {
return &conversationDatabase{
conversationDB: conversation,
cache: cache,
@@ -77,7 +78,7 @@ func NewConversationDatabase(conversation relationtb.ConversationModelInterface,
type conversationDatabase struct {
conversationDB relationtb.ConversationModelInterface
cache cache.ConversationCache
tx tx.CtxTx
tx tx.Tx
}
func (c *conversationDatabase) SetUsersConversationFieldTx(ctx context.Context, userIDs []string, conversation *relationtb.ConversationModel, fieldMap map[string]any) (err error) {
@@ -105,13 +106,13 @@ func (c *conversationDatabase) SetUsersConversationFieldTx(ctx context.Context,
cache = cache.DelConversationNotReceiveMessageUserIDs(conversation.ConversationID)
}
}
NotUserIDs := utils.DifferenceString(haveUserIDs, userIDs)
NotUserIDs := stringutil.DifferenceString(haveUserIDs, userIDs)
log.ZDebug(ctx, "SetUsersConversationFieldTx", "NotUserIDs", NotUserIDs, "haveUserIDs", haveUserIDs, "userIDs", userIDs)
var conversations []*relationtb.ConversationModel
now := time.Now()
for _, v := range NotUserIDs {
temp := new(relationtb.ConversationModel)
if err = utils.CopyStructFields(temp, conversation); err != nil {
if err = datautil.CopyStructFields(temp, conversation); err != nil {
return err
}
temp.OwnerUserID = v
@@ -205,7 +206,7 @@ func (c *conversationDatabase) GetUserAllConversation(ctx context.Context, owner
func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUserID string, conversations []*relationtb.ConversationModel) error {
return c.tx.Transaction(ctx, func(ctx context.Context) error {
cache := c.cache.NewCache()
groupIDs := utils.Distinct(utils.Filter(conversations, func(e *relationtb.ConversationModel) (string, bool) {
groupIDs := datautil.Distinct(datautil.Filter(conversations, func(e *relationtb.ConversationModel) (string, bool) {
return e.GroupID, e.GroupID != ""
}))
for _, groupID := range groupIDs {
@@ -235,7 +236,7 @@ func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUs
var notExistConversations []*relationtb.ConversationModel
for _, conversation := range conversations {
if !utils.IsContain(conversation.ConversationID, existConversationIDs) {
if !datautil.Contain(conversation.ConversationID, existConversationIDs...) {
notExistConversations = append(notExistConversations, conversation)
}
}
@@ -246,28 +247,28 @@ func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUs
}
cache = cache.DelConversationIDs(ownerUserID).
DelUserConversationIDsHash(ownerUserID).
DelConversationNotReceiveMessageUserIDs(utils.Slice(notExistConversations, func(e *relationtb.ConversationModel) string { return e.ConversationID })...)
DelConversationNotReceiveMessageUserIDs(datautil.Slice(notExistConversations, func(e *relationtb.ConversationModel) string { return e.ConversationID })...)
}
return cache.ExecDel(ctx)
})
}
//func (c *conversationDatabase) FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error) {
// func (c *conversationDatabase) FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error) {
// return c.cache.GetSuperGroupRecvMsgNotNotifyUserIDs(ctx, groupID)
//}
func (c *conversationDatabase) CreateGroupChatConversation(ctx context.Context, groupID string, userIDs []string) error {
return c.tx.Transaction(ctx, func(ctx context.Context) error {
cache := c.cache.NewCache()
conversationID := msgprocessor.GetConversationIDBySessionType(constant.SuperGroupChatType, groupID)
conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID)
existConversationUserIDs, err := c.conversationDB.FindUserID(ctx, userIDs, []string{conversationID})
if err != nil {
return err
}
notExistUserIDs := utils.DifferenceString(userIDs, existConversationUserIDs)
notExistUserIDs := stringutil.DifferenceString(userIDs, existConversationUserIDs)
var conversations []*relationtb.ConversationModel
for _, v := range notExistUserIDs {
conversation := relationtb.ConversationModel{ConversationType: constant.SuperGroupChatType, GroupID: groupID, OwnerUserID: v, ConversationID: conversationID}
conversation := relationtb.ConversationModel{ConversationType: constant.ReadGroupChatType, GroupID: groupID, OwnerUserID: v, ConversationID: conversationID}
conversations = append(conversations, &conversation)
cache = cache.DelConversations(v, conversationID).DelConversationNotReceiveMessageUserIDs(conversationID)
}
pkg/common/db/controller/doc.go
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright © 2023 OpenIM. All rights reserved.
// Copyright © 2024 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
pkg/common/db/controller/friend.go
+16 -17
View File
@@ -19,15 +19,15 @@ import (
"fmt"
"time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/mcontext"
"github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/db/tx"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/mcontext"
"github.com/openimsdk/tools/utils/datautil"
)
type FriendDatabase interface {
@@ -80,11 +80,11 @@ type FriendDatabase interface {
type friendDatabase struct {
friend relation.FriendModelInterface
friendRequest relation.FriendRequestModelInterface
tx tx.CtxTx
tx tx.Tx
cache cache.FriendCache
}
func NewFriendDatabase(friend relation.FriendModelInterface, friendRequest relation.FriendRequestModelInterface, cache cache.FriendCache, tx tx.CtxTx) FriendDatabase {
func NewFriendDatabase(friend relation.FriendModelInterface, friendRequest relation.FriendRequestModelInterface, cache cache.FriendCache, tx tx.Tx) FriendDatabase {
return &friendDatabase{friend: friend, friendRequest: friendRequest, cache: cache, tx: tx}
}
@@ -106,8 +106,8 @@ func (f *friendDatabase) CheckIn(ctx context.Context, userID1, userID2 string) (
}
// Check if userID2 is in userID1's friend list and vice versa
inUser1Friends = utils.IsContain(userID2, userID1FriendIDs)
inUser2Friends = utils.IsContain(userID1, userID2FriendIDs)
inUser1Friends = datautil.Contain(userID2, userID1FriendIDs...)
inUser2Friends = datautil.Contain(userID1, userID2FriendIDs...)
return inUser1Friends, inUser2Friends, nil
}
@@ -139,7 +139,7 @@ func (f *friendDatabase) AddFriendRequest(ctx context.Context, fromUserID, toUse
func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, addSource int32) (err error) {
return f.tx.Transaction(ctx, func(ctx context.Context) error {
cache := f.cache.NewCache()
// User find friends
// user find friends
fs1, err := f.friend.FindFriends(ctx, ownerUserID, friendUserIDs)
if err != nil {
return err
@@ -148,7 +148,7 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string,
for _, v := range friendUserIDs {
fs1 = append(fs1, &relation.FriendModel{OwnerUserID: ownerUserID, FriendUserID: v, AddSource: addSource, OperatorUserID: opUserID})
}
fs11 := utils.DistinctAny(fs1, func(e *relation.FriendModel) string {
fs11 := datautil.DistinctAny(fs1, func(e *relation.FriendModel) string {
return e.FriendUserID
})
@@ -165,7 +165,7 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string,
fs2 = append(fs2, &relation.FriendModel{OwnerUserID: v, FriendUserID: ownerUserID, AddSource: addSource, OperatorUserID: opUserID})
newFriendIDs = append(newFriendIDs, v)
}
fs22 := utils.DistinctAny(fs2, func(e *relation.FriendModel) string {
fs22 := datautil.DistinctAny(fs2, func(e *relation.FriendModel) string {
return e.OwnerUserID
})
err = f.friend.Create(ctx, fs22)
@@ -212,14 +212,13 @@ func (f *friendDatabase) RefuseFriendRequest(ctx context.Context, friendRequest
// AgreeFriendRequest accepts a friend request. It first checks for an existing, unprocessed request.
func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest *relation.FriendRequestModel) (err error) {
return f.tx.Transaction(ctx, func(ctx context.Context) error {
defer log.ZDebug(ctx, "return line")
now := time.Now()
fr, err := f.friendRequest.Take(ctx, friendRequest.FromUserID, friendRequest.ToUserID)
if err != nil {
return err
}
if fr.HandleResult != 0 {
return errs.ErrArgs.Wrap("the friend request has been processed")
return errs.ErrArgs.WrapMsg("the friend request has been processed")
}
friendRequest.HandlerUserID = mcontext.GetOpUserID(ctx)
friendRequest.HandleResult = constant.FriendResponseAgree
@@ -246,7 +245,7 @@ func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest *
if err != nil {
return err
}
existsMap := utils.SliceSet(utils.Slice(exists, func(friend *relation.FriendModel) [2]string {
existsMap := datautil.SliceSet(datautil.Slice(exists, func(friend *relation.FriendModel) [2]string {
return [...]string{friend.OwnerUserID, friend.FriendUserID} // My - Friend
}))
var adds []*relation.FriendModel
pkg/common/db/controller/group.go
+11 -9
View File
@@ -16,15 +16,16 @@ package controller
import (
"context"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/tools/utils"
"github.com/dtm-labs/rockscache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/db/tx"
"github.com/openimsdk/tools/utils/datautil"
"github.com/redis/go-redis/v9"
)
@@ -107,10 +108,11 @@ type GroupDatabase interface {
func NewGroupDatabase(
rdb redis.UniversalClient,
localCache *config.LocalCache,
groupDB relationtb.GroupModelInterface,
groupMemberDB relationtb.GroupMemberModelInterface,
groupRequestDB relationtb.GroupRequestModelInterface,
ctxTx tx.CtxTx,
ctxTx tx.Tx,
groupHash cache.GroupHash,
) GroupDatabase {
rcOptions := rockscache.NewDefaultOptions()
@@ -121,7 +123,7 @@ func NewGroupDatabase(
groupMemberDB: groupMemberDB,
groupRequestDB: groupRequestDB,
ctxTx: ctxTx,
cache: cache.NewGroupCacheRedis(rdb, groupDB, groupMemberDB, groupRequestDB, groupHash, rcOptions),
cache: cache.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash, rcOptions),
}
}
@@ -129,7 +131,7 @@ type groupDatabase struct {
groupDB relationtb.GroupModelInterface
groupMemberDB relationtb.GroupMemberModelInterface
groupRequestDB relationtb.GroupRequestModelInterface
ctxTx tx.CtxTx
ctxTx tx.Tx
cache cache.GroupCache
}
@@ -270,7 +272,7 @@ func (g *groupDatabase) PageGetJoinGroup(ctx context.Context, userID string, pag
if err != nil {
return 0, nil, err
}
for _, groupID := range utils.Paginate(groupIDs, int(pagination.GetPageNumber()), int(pagination.GetShowNumber())) {
for _, groupID := range datautil.Paginate(groupIDs, int(pagination.GetPageNumber()), int(pagination.GetShowNumber())) {
groupMembers, err := g.cache.GetGroupMembersInfo(ctx, groupID, []string{userID})
if err != nil {
return 0, nil, err
@@ -285,7 +287,7 @@ func (g *groupDatabase) PageGetGroupMember(ctx context.Context, groupID string,
if err != nil {
return 0, nil, err
}
pageIDs := utils.Paginate(groupMemberIDs, int(pagination.GetPageNumber()), int(pagination.GetShowNumber()))
pageIDs := datautil.Paginate(groupMemberIDs, int(pagination.GetPageNumber()), int(pagination.GetShowNumber()))
if len(pageIDs) == 0 {
return int64(len(groupMemberIDs)), nil, nil
}
pkg/common/db/controller/msg.go
+222 -192
View File
@@ -18,21 +18,22 @@ import (
"context"
"encoding/json"
"errors"
"strings"
"time"
"github.com/OpenIMSDK/protocol/constant"
pbmsg "github.com/OpenIMSDK/protocol/msg"
"github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/convert"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
unrelationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/common/kafka"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics"
"github.com/openimsdk/protocol/constant"
pbmsg "github.com/openimsdk/protocol/msg"
"github.com/openimsdk/protocol/sdkws"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/tools/log"
"github.com/openimsdk/tools/mq/kafka"
"github.com/openimsdk/tools/utils/datautil"
"github.com/openimsdk/tools/utils/timeutil"
"github.com/redis/go-redis/v9"
"go.mongodb.org/mongo-driver/mongo"
)
@@ -47,7 +48,7 @@ type CommonMsgDatabase interface {
// BatchInsertChat2DB inserts a batch of messages into the database for a specific conversation.
BatchInsertChat2DB(ctx context.Context, conversationID string, msgs []*sdkws.MsgData, currentMaxSeq int64) error
// RevokeMsg revokes a message in a conversation.
RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *unrelationtb.RevokeModel) error
RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *relation.RevokeModel) error
// MarkSingleChatMsgsAsRead marks messages as read for a single chat by sequence numbers.
MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, seqs []int64) error
// DeleteMessagesFromCache deletes message caches from Redis by sequence numbers.
@@ -100,75 +101,58 @@ type CommonMsgDatabase interface {
MsgToPushMQ(ctx context.Context, key, conversarionID string, msg2mq *sdkws.MsgData) (int32, int64, error)
MsgToMongoMQ(ctx context.Context, key, conversarionID string, msgs []*sdkws.MsgData, lastSeq int64) error
RangeUserSendCount(
ctx context.Context,
start time.Time,
end time.Time,
group bool,
ase bool,
pageNumber int32,
showNumber int32,
) (msgCount int64, userCount int64, users []*unrelationtb.UserCount, dateCount map[string]int64, err error)
RangeGroupSendCount(
ctx context.Context,
start time.Time,
end time.Time,
ase bool,
pageNumber int32,
showNumber int32,
) (msgCount int64, userCount int64, groups []*unrelationtb.GroupCount, dateCount map[string]int64, err error)
RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*relation.UserCount, dateCount map[string]int64, err error)
RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*relation.GroupCount, dateCount map[string]int64, err error)
ConvertMsgsDocLen(ctx context.Context, conversationIDs []string)
// clear msg
GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*relation.MsgDocModel, error)
DeleteDocMsgBefore(ctx context.Context, ts int64, doc *relation.MsgDocModel) ([]int, error)
}
func NewCommonMsgDatabase(msgDocModel unrelationtb.MsgDocModelInterface, cacheModel cache.MsgModel, config *config.GlobalConfig) (CommonMsgDatabase, error) {
producerConfig := &kafka.ProducerConfig{
ProducerAck: config.Kafka.ProducerAck,
CompressType: config.Kafka.CompressType,
Username: config.Kafka.Username,
Password: config.Kafka.Password,
}
var tlsConfig *kafka.TLSConfig
if config.Kafka.TLS != nil {
tlsConfig = &kafka.TLSConfig{
CACrt: config.Kafka.TLS.CACrt,
ClientCrt: config.Kafka.TLS.ClientCrt,
ClientKey: config.Kafka.TLS.ClientKey,
ClientKeyPwd: config.Kafka.TLS.ClientKeyPwd,
InsecureSkipVerify: false,
}
}
producerToRedis, err := kafka.NewKafkaProducer(config.Kafka.Addr, config.Kafka.LatestMsgToRedis.Topic, producerConfig, tlsConfig)
func NewCommonMsgDatabase(msgDocModel relation.MsgDocModelInterface, msg cache.MsgCache, seq cache.SeqCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) {
conf, err := kafka.BuildProducerConfig(*kafkaConf.Build())
if err != nil {
return nil, err
}
producerToMongo, err := kafka.NewKafkaProducer(config.Kafka.Addr, config.Kafka.MsgToMongo.Topic, producerConfig, tlsConfig)
producerToRedis, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToRedisTopic)
if err != nil {
return nil, err
}
producerToPush, err := kafka.NewKafkaProducer(config.Kafka.Addr, config.Kafka.MsgToPush.Topic, producerConfig, tlsConfig)
producerToMongo, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToMongoTopic)
if err != nil {
return nil, err
}
producerToPush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToPushTopic)
if err != nil {
return nil, err
}
return &commonMsgDatabase{
msgDocDatabase: msgDocModel,
cache: cacheModel,
msg: msg,
seq: seq,
producer: producerToRedis,
producerToMongo: producerToMongo,
producerToPush: producerToPush,
}, nil
}
func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database, config *config.GlobalConfig) (CommonMsgDatabase, error) {
cacheModel := cache.NewMsgCacheModel(rdb, config)
msgDocModel := unrelation.NewMsgMongoDriver(database)
return NewCommonMsgDatabase(msgDocModel, cacheModel, config)
}
//func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database, config *tools.CronTaskConfig) (CommonMsgDatabase, error) {
// msgDocModel, err := mgo.NewMsgMongo(database)
// if err != nil {
// return nil, err
// }
// //todo MsgCacheTimeout
// msg := cache.NewMsgCache(rdb, 86400, config.RedisConfig.EnablePipeline)
// seq := cache.NewSeqCache(rdb)
// return NewCommonMsgDatabase(msgDocModel, msg, seq, &config.KafkaConfig)
//}
type commonMsgDatabase struct {
msgDocDatabase unrelationtb.MsgDocModelInterface
msg unrelationtb.MsgDocModel
cache cache.MsgModel
msgDocDatabase relation.MsgDocModelInterface
msgTable relation.MsgDocModel
msg cache.MsgCache
seq cache.SeqCache
producer *kafka.Producer
producerToMongo *kafka.Producer
producerToModify *kafka.Producer
@@ -209,24 +193,24 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI
if len(fields) == 0 {
return nil
}
num := db.msg.GetSingleGocMsgNum()
num := db.msgTable.GetSingleGocMsgNum()
// num = 100
for i, field := range fields { // Check the type of the field
var ok bool
switch key {
case updateKeyMsg:
var msg *unrelationtb.MsgDataModel
msg, ok = field.(*unrelationtb.MsgDataModel)
var msg *relation.MsgDataModel
msg, ok = field.(*relation.MsgDataModel)
if msg != nil && msg.Seq != firstSeq+int64(i) {
return errs.ErrInternalServer.Wrap("seq is invalid")
return errs.ErrInternalServer.WrapMsg("seq is invalid")
}
case updateKeyRevoke:
_, ok = field.(*unrelationtb.RevokeModel)
_, ok = field.(*relation.RevokeModel)
default:
return errs.ErrInternalServer.Wrap("key is invalid")
return errs.ErrInternalServer.WrapMsg("key is invalid")
}
if !ok {
return errs.ErrInternalServer.Wrap("field type is invalid")
return errs.ErrInternalServer.WrapMsg("field type is invalid")
}
}
// Returns true if the document exists in the database, false if the document does not exist in the database
@@ -235,8 +219,8 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI
res *mongo.UpdateResult
err error
)
docID := db.msg.GetDocID(conversationID, seq)
index := db.msg.GetMsgIndex(seq)
docID := db.msgTable.GetDocID(conversationID, seq)
index := db.msgTable.GetMsgIndex(seq)
field := fields[i]
switch key {
case updateKeyMsg:
@@ -261,31 +245,31 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI
continue // The current data has been updated, skip the current data
}
}
doc := unrelationtb.MsgDocModel{
DocID: db.msg.GetDocID(conversationID, seq),
Msg: make([]*unrelationtb.MsgInfoModel, num),
doc := relation.MsgDocModel{
DocID: db.msgTable.GetDocID(conversationID, seq),
Msg: make([]*relation.MsgInfoModel, num),
}
var insert int // Inserted data number
for j := i; j < len(fields); j++ {
seq = firstSeq + int64(j)
if db.msg.GetDocID(conversationID, seq) != doc.DocID {
if db.msgTable.GetDocID(conversationID, seq) != doc.DocID {
break
}
insert++
switch key {
case updateKeyMsg:
doc.Msg[db.msg.GetMsgIndex(seq)] = &unrelationtb.MsgInfoModel{
Msg: fields[j].(*unrelationtb.MsgDataModel),
doc.Msg[db.msgTable.GetMsgIndex(seq)] = &relation.MsgInfoModel{
Msg: fields[j].(*relation.MsgDataModel),
}
case updateKeyRevoke:
doc.Msg[db.msg.GetMsgIndex(seq)] = &unrelationtb.MsgInfoModel{
Revoke: fields[j].(*unrelationtb.RevokeModel),
doc.Msg[db.msgTable.GetMsgIndex(seq)] = &relation.MsgInfoModel{
Revoke: fields[j].(*relation.RevokeModel),
}
}
}
for i, model := range doc.Msg {
if model == nil {
model = &unrelationtb.MsgInfoModel{}
model = &relation.MsgInfoModel{}
doc.Msg[i] = model
}
if model.DelList == nil {
@@ -308,16 +292,16 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI
func (db *commonMsgDatabase) BatchInsertChat2DB(ctx context.Context, conversationID string, msgList []*sdkws.MsgData, currentMaxSeq int64) error {
if len(msgList) == 0 {
return errs.ErrArgs.Wrap("msgList is empty")
return errs.ErrArgs.WrapMsg("msgList is empty")
}
msgs := make([]any, len(msgList))
for i, msg := range msgList {
if msg == nil {
continue
}
var offlinePushModel *unrelationtb.OfflinePushModel
var offlinePushModel *relation.OfflinePushModel
if msg.OfflinePushInfo != nil {
offlinePushModel = &unrelationtb.OfflinePushModel{
offlinePushModel = &relation.OfflinePushModel{
Title: msg.OfflinePushInfo.Title,
Desc: msg.OfflinePushInfo.Desc,
Ex: msg.OfflinePushInfo.Ex,
@@ -325,7 +309,7 @@ func (db *commonMsgDatabase) BatchInsertChat2DB(ctx context.Context, conversatio
IOSBadgeCount: msg.OfflinePushInfo.IOSBadgeCount,
}
}
msgs[i] = &unrelationtb.MsgDataModel{
msgs[i] = &relation.MsgDataModel{
SendID: msg.SendID,
RecvID: msg.RecvID,
GroupID: msg.GroupID,
@@ -352,15 +336,15 @@ func (db *commonMsgDatabase) BatchInsertChat2DB(ctx context.Context, conversatio
return db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq)
}
func (db *commonMsgDatabase) RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *unrelationtb.RevokeModel) error {
func (db *commonMsgDatabase) RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *relation.RevokeModel) error {
return db.BatchInsertBlock(ctx, conversationID, []any{revoke}, updateKeyRevoke, seq)
}
func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, totalSeqs []int64) error {
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, totalSeqs) {
for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, totalSeqs) {
var indexes []int64
for _, seq := range seqs {
indexes = append(indexes, db.msg.GetMsgIndex(seq))
indexes = append(indexes, db.msgTable.GetMsgIndex(seq))
}
log.ZDebug(ctx, "MarkSingleChatMsgsAsRead", "userID", userID, "docID", docID, "indexes", indexes)
if err := db.msgDocDatabase.MarkSingleChatMsgsAsRead(ctx, userID, docID, indexes); err != nil {
@@ -372,25 +356,25 @@ func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userI
}
func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error {
return db.cache.DeleteMessages(ctx, conversationID, seqs)
return db.msg.DeleteMessages(ctx, conversationID, seqs)
}
func (db *commonMsgDatabase) DelUserDeleteMsgsList(ctx context.Context, conversationID string, seqs []int64) {
db.cache.DelUserDeleteMsgsList(ctx, conversationID, seqs)
db.msg.DelUserDeleteMsgsList(ctx, conversationID, seqs)
}
func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) {
currentMaxSeq, err := db.cache.GetMaxSeq(ctx, conversationID)
currentMaxSeq, err := db.seq.GetMaxSeq(ctx, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
log.ZError(ctx, "db.cache.GetMaxSeq", err)
log.ZError(ctx, "db.seq.GetMaxSeq", err)
return 0, false, err
}
lenList := len(msgs)
if int64(lenList) > db.msg.GetSingleGocMsgNum() {
return 0, false, errors.New("too large")
if int64(lenList) > db.msgTable.GetSingleGocMsgNum() {
return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap()
}
if lenList < 1 {
return 0, false, errors.New("too short as 0")
return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap()
}
if errs.Unwrap(err) == redis.Nil {
isNew = true
@@ -402,19 +386,22 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa
m.Seq = currentMaxSeq
userSeqMap[m.SendID] = m.Seq
}
failedNum, err := db.cache.SetMessageToCache(ctx, conversationID, msgs)
failedNum, err := db.msg.SetMessageToCache(ctx, conversationID, msgs)
if err != nil {
prommetrics.MsgInsertRedisFailedCounter.Add(float64(failedNum))
log.ZError(ctx, "setMessageToCache error", err, "len", len(msgs), "conversationID", conversationID)
} else {
prommetrics.MsgInsertRedisSuccessCounter.Inc()
}
err = db.cache.SetMaxSeq(ctx, conversationID, currentMaxSeq)
err = db.seq.SetMaxSeq(ctx, conversationID, currentMaxSeq)
if err != nil {
log.ZError(ctx, "db.cache.SetMaxSeq error", err, "conversationID", conversationID)
log.ZError(ctx, "db.seq.SetMaxSeq error", err, "conversationID", conversationID)
prommetrics.SeqSetFailedCounter.Inc()
}
err = db.cache.SetHasReadSeqs(ctx, conversationID, userSeqMap)
err = db.seq.SetHasReadSeqs(ctx, conversationID, userSeqMap)
if err != nil {
log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID)
prommetrics.SeqSetFailedCounter.Inc()
@@ -423,7 +410,7 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa
}
func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversationID string, seqs []int64) (totalMsgs []*sdkws.MsgData, err error) {
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, seqs) {
for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) {
// log.ZDebug(ctx, "getMsgBySeqs", "docID", docID, "seqs", seqs)
msgs, err := db.findMsgInfoBySeq(ctx, userID, docID, conversationID, seqs)
if err != nil {
@@ -436,7 +423,7 @@ func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversat
return totalMsgs, nil
}
func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][]*unrelationtb.MsgInfoModel, userID, conversationID string, msg *unrelationtb.MsgInfoModel) {
func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][]*relation.MsgInfoModel, userID, conversationID string, msg *relation.MsgInfoModel) {
if msg.IsRead {
msg.Msg.IsRead = true
}
@@ -458,12 +445,12 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][
if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.ContentType == constant.MsgRevokeNotification {
return
}
var msgs []*unrelationtb.MsgInfoModel
var msgs []*relation.MsgInfoModel
if v, ok := cache[quoteMsg.QuoteMessage.Seq]; ok {
msgs = v
} else {
if quoteMsg.QuoteMessage.Seq > 0 {
ms, err := db.msgDocDatabase.GetMsgBySeqIndexIn1Doc(ctx, userID, db.msg.GetDocID(conversationID, quoteMsg.QuoteMessage.Seq), []int64{quoteMsg.QuoteMessage.Seq})
ms, err := db.msgDocDatabase.GetMsgBySeqIndexIn1Doc(ctx, userID, db.msgTable.GetDocID(conversationID, quoteMsg.QuoteMessage.Seq), []int64{quoteMsg.QuoteMessage.Seq})
if err != nil {
log.ZError(ctx, "GetMsgBySeqIndexIn1Doc", err, "conversationID", conversationID, "seq", quoteMsg.QuoteMessage.Seq)
return
@@ -487,17 +474,17 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][
return
}
msg.Msg.Content = string(data)
if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msg.GetDocID(conversationID, msg.Msg.Seq), db.msg.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil {
if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msgTable.GetDocID(conversationID, msg.Msg.Seq), db.msgTable.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil {
log.ZError(ctx, "UpdateMsgContent", err)
}
}
func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, conversationID string, seqs []int64) (totalMsgs []*unrelationtb.MsgInfoModel, err error) {
func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, conversationID string, seqs []int64) (totalMsgs []*relation.MsgInfoModel, err error) {
msgs, err := db.msgDocDatabase.GetMsgBySeqIndexIn1Doc(ctx, userID, docID, seqs)
if err != nil {
return nil, err
}
tempCache := make(map[int64][]*unrelationtb.MsgInfoModel)
tempCache := make(map[int64][]*relation.MsgInfoModel)
for _, msg := range msgs {
db.handlerDBMsg(ctx, tempCache, userID, conversationID, msg)
}
@@ -506,7 +493,7 @@ func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID
func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID string, conversationID string, allSeqs []int64, begin, end int64) (seqMsgs []*sdkws.MsgData, err error) {
log.ZDebug(ctx, "getMsgBySeqsRange", "conversationID", conversationID, "allSeqs", allSeqs, "begin", begin, "end", end)
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, allSeqs) {
for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, allSeqs) {
log.ZDebug(ctx, "getMsgBySeqsRange", "docID", docID, "seqs", seqs)
msgs, err := db.findMsgInfoBySeq(ctx, userID, docID, conversationID, seqs)
if err != nil {
@@ -542,23 +529,23 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID strin
// "userMinSeq" can be set as the same value as the conversation's "maxSeq" at the moment they join the group.
// This ensures that their message retrieval starts from the point they joined.
func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) {
userMinSeq, err := db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
userMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
minSeq, err := db.cache.GetMinSeq(ctx, conversationID)
minSeq, err := db.seq.GetMinSeq(ctx, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
if userMinSeq > minSeq {
minSeq = userMinSeq
}
//"minSeq" represents the startSeq value that the user can retrieve.
// "minSeq" represents the startSeq value that the user can retrieve.
if minSeq > end {
log.ZInfo(ctx, "minSeq > end", "minSeq", minSeq, "end", end)
log.ZWarn(ctx, "minSeq > end", errs.New("minSeq>end"), "minSeq", minSeq, "end", end)
return 0, 0, nil, nil
}
maxSeq, err := db.cache.GetMaxSeq(ctx, conversationID)
maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
@@ -568,7 +555,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
maxSeq = userMaxSeq
}
}
//"maxSeq" represents the endSeq value that the user can retrieve.
// "maxSeq" represents the endSeq value that the user can retrieve.
if begin < minSeq {
begin = minSeq
@@ -576,9 +563,9 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
if end > maxSeq {
end = maxSeq
}
//"begin" and "end" represent the actual startSeq and endSeq values that the user can retrieve.
// "begin" and "end" represent the actual startSeq and endSeq values that the user can retrieve.
if end < begin {
return 0, 0, nil, errs.ErrArgs.Wrap("seq end < begin")
return 0, 0, nil, errs.ErrArgs.WrapMsg("seq end < begin")
}
var seqs []int64
if end-begin+1 <= num {
@@ -591,25 +578,13 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
}
}
//167 178 10
//if end-num < {
//
//}
//var seqs []int64
//for i := end; i > end-num; i-- {
// if i >= begin {
// seqs = append([]int64{i}, seqs...)
// } else {
// break
// }
//}
if len(seqs) == 0 {
return 0, 0, nil, nil
}
newBegin := seqs[0]
newEnd := seqs[len(seqs)-1]
log.ZDebug(ctx, "GetMsgBySeqsRange", "first seqs", seqs, "newBegin", newBegin, "newEnd", newEnd)
cachedMsgs, failedSeqs, err := db.cache.GetMessagesBySeq(ctx, conversationID, seqs)
cachedMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, seqs)
if err != nil {
if err != redis.Nil {
@@ -618,13 +593,13 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
}
var successMsgs []*sdkws.MsgData
if len(cachedMsgs) > 0 {
delSeqs, err := db.cache.GetUserDelList(ctx, userID, conversationID)
delSeqs, err := db.msg.GetUserDelList(ctx, userID, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
var cacheDelNum int
for _, msg := range cachedMsgs {
if !utils.Contain(msg.Seq, delSeqs...) {
if !datautil.Contain(msg.Seq, delSeqs...) {
successMsgs = append(successMsgs, msg)
} else {
cacheDelNum += 1
@@ -635,7 +610,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
for i := 1; i <= cacheDelNum; {
newSeq := newBegin - int64(i)
if newSeq >= begin {
if !utils.Contain(newSeq, delSeqs...) {
if !datautil.Contain(newSeq, delSeqs...) {
log.ZDebug(ctx, "seq del in cache, a new seq in range append", "new seq", newSeq)
reGetSeqsCache = append(reGetSeqsCache, newSeq)
i++
@@ -646,7 +621,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
}
if len(reGetSeqsCache) > 0 {
log.ZDebug(ctx, "reGetSeqsCache", "reGetSeqsCache", reGetSeqsCache)
cachedMsgs, failedSeqs2, err := db.cache.GetMessagesBySeq(ctx, conversationID, reGetSeqsCache)
cachedMsgs, failedSeqs2, err := db.msg.GetMessagesBySeq(ctx, conversationID, reGetSeqsCache)
if err != nil {
if err != redis.Nil {
@@ -676,15 +651,15 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin
}
func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (int64, int64, []*sdkws.MsgData, error) {
userMinSeq, err := db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
userMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
minSeq, err := db.cache.GetMinSeq(ctx, conversationID)
minSeq, err := db.seq.GetMinSeq(ctx, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
maxSeq, err := db.cache.GetMaxSeq(ctx, conversationID)
maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
@@ -697,28 +672,14 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co
newSeqs = append(newSeqs, seq)
}
}
successMsgs, failedSeqs, err := db.cache.GetMessagesBySeq(ctx, conversationID, newSeqs)
successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs)
if err != nil {
if err != redis.Nil {
log.ZError(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID)
}
}
log.ZInfo(
ctx,
"db.cache.GetMessagesBySeq",
"userID",
userID,
"conversationID",
conversationID,
"seqs",
seqs,
"successMsgs",
len(successMsgs),
"failedSeqs",
failedSeqs,
"conversationID",
conversationID,
)
log.ZDebug(ctx, "db.seq.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs",
seqs, "len(successMsgs)", len(successMsgs), "failedSeqs", failedSeqs)
if len(failedSeqs) > 0 {
mongoMsgs, err := db.getMsgBySeqs(ctx, userID, conversationID, failedSeqs)
@@ -739,17 +700,17 @@ func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Cont
if err != nil {
return err
}
log.ZInfo(ctx, "DeleteConversationMsgsAndSetMinSeq", "conversationID", conversationID, "minSeq", minSeq)
log.ZDebug(ctx, "DeleteConversationMsgsAndSetMinSeq", "conversationID", conversationID, "minSeq", minSeq)
if minSeq == 0 {
return nil
}
if remainTime == 0 {
err = db.cache.CleanUpOneConversationAllMsg(ctx, conversationID)
err = db.msg.CleanUpOneConversationAllMsg(ctx, conversationID)
if err != nil {
log.ZWarn(ctx, "CleanUpOneUserAllMsg", err, "conversationID", conversationID)
}
}
return db.cache.SetMinSeq(ctx, conversationID, minSeq)
return db.seq.SetMinSeq(ctx, conversationID, minSeq)
}
func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error) {
@@ -759,7 +720,7 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string
msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1)
if err != nil || msgDocModel.DocID == "" {
if err != nil {
if err == unrelation.ErrMsgListNotExist {
if err == relation.ErrMsgListNotExist {
log.ZDebug(ctx, "not doc find", "conversationID", conversationID, "userID", userID, "index", index)
} else {
log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index)
@@ -769,14 +730,14 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string
break
}
index++
//&& msgDocModel.Msg[0].Msg.SendTime > lastMsgDestructTime.UnixMilli()
// && msgDocModel.Msg[0].Msg.SendTime > lastMsgDestructTime.UnixMilli()
if len(msgDocModel.Msg) > 0 {
i := 0
var over bool
for _, msg := range msgDocModel.Msg {
i++
if msg != nil && msg.Msg != nil && msg.Msg.SendTime+destructTime*1000 <= time.Now().UnixMilli() {
if msg.Msg.SendTime+destructTime*1000 > lastMsgDestructTime.UnixMilli() && !utils.Contain(userID, msg.DelList...) {
if msg.Msg.SendTime+destructTime*1000 > lastMsgDestructTime.UnixMilli() && !datautil.Contain(userID, msg.DelList...) {
seqs = append(seqs, msg.Msg.Seq)
}
} else {
@@ -794,12 +755,12 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string
log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversationID, "userID", userID, "seqs", seqs)
if len(seqs) > 0 {
userMinSeq := seqs[len(seqs)-1] + 1
currentUserMinSeq, err := db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
currentUserMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return nil, err
}
if currentUserMinSeq < userMinSeq {
if err := db.cache.SetConversationUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil {
if err := db.seq.SetConversationUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil {
return nil, err
}
}
@@ -826,7 +787,7 @@ func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversatio
msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1)
if err != nil || msgDocModel.DocID == "" {
if err != nil {
if err == unrelation.ErrMsgListNotExist {
if err == relation.ErrMsgListNotExist {
log.ZDebug(ctx, "deleteMsgRecursion ErrMsgListNotExist", "conversationID", conversationID, "index:", index)
} else {
log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index)
@@ -840,10 +801,10 @@ func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversatio
return delStruct.getSetMinSeq() + 1, nil
}
log.ZDebug(ctx, "doc info", "conversationID", conversationID, "index", index, "docID", msgDocModel.DocID, "len", len(msgDocModel.Msg))
if int64(len(msgDocModel.Msg)) > db.msg.GetSingleGocMsgNum() {
if int64(len(msgDocModel.Msg)) > db.msgTable.GetSingleGocMsgNum() {
log.ZWarn(ctx, "msgs too large", nil, "length", len(msgDocModel.Msg), "docID:", msgDocModel.DocID)
}
if msgDocModel.IsFull() && msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < utils.GetCurrentTimestampByMill() {
if msgDocModel.IsFull() && msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < timeutil.GetCurrentTimestampByMill() {
log.ZDebug(ctx, "doc is full and all msg is expired", "docID", msgDocModel.DocID)
delStruct.delDocIDs = append(delStruct.delDocIDs, msgDocModel.DocID)
delStruct.minSeq = msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.Seq
@@ -851,7 +812,7 @@ func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversatio
var delMsgIndexs []int
for i, MsgInfoModel := range msgDocModel.Msg {
if MsgInfoModel != nil && MsgInfoModel.Msg != nil {
if utils.GetCurrentTimestampByMill() > MsgInfoModel.Msg.SendTime+(remainTime*1000) {
if timeutil.GetCurrentTimestampByMill() > MsgInfoModel.Msg.SendTime+(remainTime*1000) {
delMsgIndexs = append(delMsgIndexs, i)
}
}
@@ -868,13 +829,13 @@ func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversatio
}
func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, allSeqs []int64) error {
if err := db.cache.DeleteMessages(ctx, conversationID, allSeqs); err != nil {
if err := db.msg.DeleteMessages(ctx, conversationID, allSeqs); err != nil {
return err
}
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, allSeqs) {
for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, allSeqs) {
var indexes []int
for _, seq := range seqs {
indexes = append(indexes, int(db.msg.GetMsgIndex(seq)))
indexes = append(indexes, int(db.msgTable.GetMsgIndex(seq)))
}
if err := db.msgDocDatabase.DeleteMsgsInOneDocByIndex(ctx, docID, indexes); err != nil {
return err
@@ -884,7 +845,7 @@ func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conve
}
func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error {
cachedMsgs, _, err := db.cache.GetMessagesBySeq(ctx, conversationID, seqs)
cachedMsgs, _, err := db.msg.GetMessagesBySeq(ctx, conversationID, seqs)
if err != nil && errs.Unwrap(err) != redis.Nil {
log.ZWarn(ctx, "DeleteUserMsgsBySeqs", err, "conversationID", conversationID, "seqs", seqs)
return err
@@ -894,14 +855,14 @@ func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID st
for _, msg := range cachedMsgs {
cacheSeqs = append(cacheSeqs, msg.Seq)
}
if err := db.cache.UserDeleteMsgs(ctx, conversationID, cacheSeqs, userID); err != nil {
if err := db.msg.UserDeleteMsgs(ctx, conversationID, cacheSeqs, userID); err != nil {
return err
}
}
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, seqs) {
for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) {
for _, seq := range seqs {
if _, err := db.msgDocDatabase.PushUnique(ctx, docID, db.msg.GetMsgIndex(seq), "del_list", []string{userID}); err != nil {
if _, err := db.msgDocDatabase.PushUnique(ctx, docID, db.msgTable.GetMsgIndex(seq), "del_list", []string{userID}); err != nil {
return err
}
}
@@ -915,91 +876,91 @@ func (db *commonMsgDatabase) DeleteMsgsBySeqs(ctx context.Context, conversationI
func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, user string, conversationIDs []string) {
for _, conversationID := range conversationIDs {
maxSeq, err := db.cache.GetMaxSeq(ctx, conversationID)
maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID)
if err != nil {
if err == redis.Nil {
log.ZInfo(ctx, "max seq is nil", "conversationID", conversationID)
log.ZDebug(ctx, "max seq is nil", "conversationID", conversationID)
} else {
log.ZError(ctx, "get max seq failed", err, "conversationID", conversationID)
}
continue
}
if err := db.cache.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil {
if err := db.seq.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil {
log.ZError(ctx, "set min seq failed", err, "conversationID", conversationID, "minSeq", maxSeq+1)
}
}
}
func (db *commonMsgDatabase) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error {
return db.cache.SetMaxSeq(ctx, conversationID, maxSeq)
return db.seq.SetMaxSeq(ctx, conversationID, maxSeq)
}
func (db *commonMsgDatabase) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) {
return db.cache.GetMaxSeqs(ctx, conversationIDs)
return db.seq.GetMaxSeqs(ctx, conversationIDs)
}
func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) {
return db.cache.GetMaxSeq(ctx, conversationID)
return db.seq.GetMaxSeq(ctx, conversationID)
}
func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error {
return db.cache.SetMinSeq(ctx, conversationID, minSeq)
return db.seq.SetMinSeq(ctx, conversationID, minSeq)
}
func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error {
return db.cache.SetMinSeqs(ctx, seqs)
return db.seq.SetMinSeqs(ctx, seqs)
}
func (db *commonMsgDatabase) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) {
return db.cache.GetMinSeqs(ctx, conversationIDs)
return db.seq.GetMinSeqs(ctx, conversationIDs)
}
func (db *commonMsgDatabase) GetMinSeq(ctx context.Context, conversationID string) (int64, error) {
return db.cache.GetMinSeq(ctx, conversationID)
return db.seq.GetMinSeq(ctx, conversationID)
}
func (db *commonMsgDatabase) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
return db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
return db.seq.GetConversationUserMinSeq(ctx, conversationID, userID)
}
func (db *commonMsgDatabase) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) {
return db.cache.GetConversationUserMinSeqs(ctx, conversationID, userIDs)
return db.seq.GetConversationUserMinSeqs(ctx, conversationID, userIDs)
}
func (db *commonMsgDatabase) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error {
return db.cache.SetConversationUserMinSeq(ctx, conversationID, userID, minSeq)
return db.seq.SetConversationUserMinSeq(ctx, conversationID, userID, minSeq)
}
func (db *commonMsgDatabase) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) {
return db.cache.SetConversationUserMinSeqs(ctx, conversationID, seqs)
return db.seq.SetConversationUserMinSeqs(ctx, conversationID, seqs)
}
func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error {
return db.cache.SetUserConversationsMinSeqs(ctx, userID, seqs)
return db.seq.SetUserConversationsMinSeqs(ctx, userID, seqs)
}
func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error {
return db.cache.UserSetHasReadSeqs(ctx, userID, hasReadSeqs)
return db.seq.UserSetHasReadSeqs(ctx, userID, hasReadSeqs)
}
func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error {
return db.cache.SetHasReadSeq(ctx, userID, conversationID, hasReadSeq)
return db.seq.SetHasReadSeq(ctx, userID, conversationID, hasReadSeq)
}
func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) {
return db.cache.GetHasReadSeqs(ctx, userID, conversationIDs)
return db.seq.GetHasReadSeqs(ctx, userID, conversationIDs)
}
func (db *commonMsgDatabase) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) {
return db.cache.GetHasReadSeq(ctx, userID, conversationID)
return db.seq.GetHasReadSeq(ctx, userID, conversationID)
}
func (db *commonMsgDatabase) SetSendMsgStatus(ctx context.Context, id string, status int32) error {
return db.cache.SetSendMsgStatus(ctx, id, status)
return db.msg.SetSendMsgStatus(ctx, id, status)
}
func (db *commonMsgDatabase) GetSendMsgStatus(ctx context.Context, id string) (int32, error) {
return db.cache.GetSendMsgStatus(ctx, id)
return db.msg.GetSendMsgStatus(ctx, id)
}
func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) {
@@ -1007,11 +968,11 @@ func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context
if err != nil {
return
}
minSeqCache, err = db.cache.GetMinSeq(ctx, conversationID)
minSeqCache, err = db.seq.GetMinSeq(ctx, conversationID)
if err != nil {
return
}
maxSeqCache, err = db.cache.GetMaxSeq(ctx, conversationID)
maxSeqCache, err = db.seq.GetMaxSeq(ctx, conversationID)
if err != nil {
return
}
@@ -1044,7 +1005,7 @@ func (db *commonMsgDatabase) RangeUserSendCount(
ase bool,
pageNumber int32,
showNumber int32,
) (msgCount int64, userCount int64, users []*unrelationtb.UserCount, dateCount map[string]int64, err error) {
) (msgCount int64, userCount int64, users []*relation.UserCount, dateCount map[string]int64, err error) {
return db.msgDocDatabase.RangeUserSendCount(ctx, start, end, group, ase, pageNumber, showNumber)
}
@@ -1055,7 +1016,7 @@ func (db *commonMsgDatabase) RangeGroupSendCount(
ase bool,
pageNumber int32,
showNumber int32,
) (msgCount int64, userCount int64, groups []*unrelationtb.GroupCount, dateCount map[string]int64, err error) {
) (msgCount int64, userCount int64, groups []*relation.GroupCount, dateCount map[string]int64, err error) {
return db.msgDocDatabase.RangeGroupSendCount(ctx, start, end, ase, pageNumber, showNumber)
}
@@ -1078,12 +1039,12 @@ func (db *commonMsgDatabase) FindOneByDocIDs(ctx context.Context, conversationID
totalMsgs := make(map[string]*sdkws.MsgData)
for _, conversationID := range conversationIDs {
seq := seqs[conversationID]
docID := db.msg.GetDocID(conversationID, seq)
docID := db.msgTable.GetDocID(conversationID, seq)
msgs, err := db.msgDocDatabase.FindOneByDocID(ctx, docID)
if err != nil {
return nil, err
}
index := db.msg.GetMsgIndex(seq)
index := db.msgTable.GetMsgIndex(seq)
totalMsgs[conversationID] = convert.MsgDB2Pb(msgs.Msg[index].Msg)
}
return totalMsgs, nil
@@ -1092,3 +1053,72 @@ func (db *commonMsgDatabase) FindOneByDocIDs(ctx context.Context, conversationID
func (db *commonMsgDatabase) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) {
db.msgDocDatabase.ConvertMsgsDocLen(ctx, conversationIDs)
}
func (db *commonMsgDatabase) GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*relation.MsgDocModel, error) {
return db.msgDocDatabase.GetBeforeMsg(ctx, ts, limit)
}
func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *relation.MsgDocModel) ([]int, error) {
var notNull int
index := make([]int, 0, len(doc.Msg))
for i, message := range doc.Msg {
if message.Msg != nil {
notNull++
if message.Msg.SendTime < ts {
index = append(index, i)
}
}
}
if len(index) == 0 {
return index, nil
}
maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq
conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")]
if err := db.setMinSeq(ctx, conversationID, maxSeq+1); err != nil {
return index, err
}
if len(index) == notNull {
return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID)
} else {
return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index)
}
}
//func (db *commonMsgDatabase) ClearMsg(ctx context.Context, ts int64) (err error) {
// var (
// docNum int
// msgNum int
// start = time.Now()
// )
// for {
// msgs, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, 100)
// if err != nil {
// return err
// }
// if len(msgs) == 0 {
// return nil
// }
// for _, msg := range msgs {
// num, err := db.deleteOneMsg(ctx, ts, msg)
// if err != nil {
// return err
// }
// docNum++
// msgNum += num
// }
// }
//}
func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID string, seq int64) error {
dbSeq, err := db.seq.GetMinSeq(ctx, conversationID)
if err != nil {
if errors.Is(errs.Unwrap(err), redis.Nil) {
return nil
}
return err
}
if dbSeq >= seq {
return nil
}
return db.seq.SetMinSeq(ctx, conversationID, seq)
}
pkg/common/db/controller/msg_test.go
-250
View File
@@ -1,250 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controller
import (
"context"
"fmt"
"math/rand"
"strconv"
"sync"
"testing"
"time"
"github.com/OpenIMSDK/tools/log"
"go.mongodb.org/mongo-driver/bson"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
unrelationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
)
func Test_BatchInsertChat2DB(t *testing.T) {
conf := config.NewGlobalConfig()
conf.Mongo.Address = []string{"192.168.44.128:37017"}
// conf.Mongo.Timeout = 60
conf.Mongo.Database = "openIM"
// conf.Mongo.Source = "admin"
conf.Mongo.Username = "root"
conf.Mongo.Password = "openIM123"
conf.Mongo.MaxPoolSize = 100
conf.RetainChatRecords = 3650
conf.ChatRecordsClearTime = "0 2 * * 3"
mongo, err := unrelation.NewMongo(conf)
if err != nil {
t.Fatal(err)
}
err = mongo.GetDatabase(conf.Mongo.Database).Client().Ping(context.Background(), nil)
if err != nil {
panic(err)
}
db := &commonMsgDatabase{
msgDocDatabase: unrelation.NewMsgMongoDriver(mongo.GetDatabase(conf.Mongo.Database)),
}
//ctx := context.Background()
//msgs := make([]*sdkws.MsgData, 0, 1)
//for i := 0; i < cap(msgs); i++ {
// msgs = append(msgs, &sdkws.MsgData{
// Content: []byte(fmt.Sprintf("test-%d", i)),
// SendTime: time.Now().UnixMilli(),
// })
//}
//err = db.BatchInsertChat2DB(ctx, "test", msgs, 0)
//if err != nil {
// panic(err)
//}
_ = db.BatchInsertChat2DB
c := mongo.GetDatabase(conf.Mongo.Database).Collection("msg")
ch := make(chan int)
rand.Seed(time.Now().UnixNano())
index := 10
var wg sync.WaitGroup
for i := 0; i < 1000; i++ {
wg.Add(1)
go func(channelID int) {
defer wg.Done()
<-ch
var arr []string
for i := 0; i < 500; i++ {
arr = append(arr, strconv.Itoa(i+1))
}
rand.Shuffle(len(arr), func(i, j int) {
arr[i], arr[j] = arr[j], arr[i]
})
for j, s := range arr {
if j == 0 {
fmt.Printf("channnelID: %d, arr[0]: %s\n", channelID, arr[j])
}
filter := bson.M{"doc_id": "test:0"}
update := bson.M{
"$addToSet": bson.M{
fmt.Sprintf("msgs.%d.del_list", index): bson.M{"$each": []string{s}},
},
}
_, err := c.UpdateOne(context.Background(), filter, update)
if err != nil {
t.Fatal(err)
}
}
}(i)
}
for i := 0; i < 1000; i++ {
wg.Add(1)
go func() {
defer wg.Done()
<-ch
var arr []string
for i := 0; i < 500; i++ {
arr = append(arr, strconv.Itoa(1001+i))
}
rand.Shuffle(len(arr), func(i, j int) {
arr[i], arr[j] = arr[j], arr[i]
})
for _, s := range arr {
filter := bson.M{"doc_id": "test:0"}
update := bson.M{
"$addToSet": bson.M{
fmt.Sprintf("msgs.%d.read_list", index): bson.M{"$each": []string{s}},
},
}
_, err := c.UpdateOne(context.Background(), filter, update)
if err != nil {
t.Fatal(err)
}
}
}()
}
time.Sleep(time.Second * 2)
close(ch)
wg.Wait()
}
func GetDB() *commonMsgDatabase {
conf := config.NewGlobalConfig()
conf.Mongo.Address = []string{"203.56.175.233:37017"}
// conf.Mongo.Timeout = 60
conf.Mongo.Database = "openim_v3"
// conf.Mongo.Source = "admin"
conf.Mongo.Username = "root"
conf.Mongo.Password = "openIM123"
conf.Mongo.MaxPoolSize = 100
conf.RetainChatRecords = 3650
conf.ChatRecordsClearTime = "0 2 * * 3"
mongo, err := unrelation.NewMongo(conf)
if err != nil {
panic(err)
}
err = mongo.GetDatabase(conf.Mongo.Database).Client().Ping(context.Background(), nil)
if err != nil {
panic(err)
}
return &commonMsgDatabase{
msgDocDatabase: unrelation.NewMsgMongoDriver(mongo.GetDatabase(conf.Mongo.Database)),
}
}
func Test_Insert(t *testing.T) {
db := GetDB()
ctx := context.Background()
var arr []any
for i := 0; i < 345; i++ {
if i%2 == 0 {
arr = append(arr, (*unrelationtb.MsgDataModel)(nil))
continue
}
arr = append(arr, &unrelationtb.MsgDataModel{
Seq: int64(i),
Content: fmt.Sprintf("test-%d", i),
})
}
if err := db.BatchInsertBlock(ctx, "test", arr, updateKeyMsg, 1); err != nil {
t.Fatal(err)
}
}
func Test_Revoke(t *testing.T) {
db := GetDB()
ctx := context.Background()
var arr []any
for i := 0; i < 456; i++ {
arr = append(arr, &unrelationtb.RevokeModel{
UserID: "uid_" + strconv.Itoa(i),
Nickname: "uname_" + strconv.Itoa(i),
Time: time.Now().UnixMilli(),
})
}
if err := db.BatchInsertBlock(ctx, "test", arr, updateKeyRevoke, 123); err != nil {
t.Fatal(err)
}
}
func Test_FindBySeq(t *testing.T) {
if err := log.InitFromConfig("", "", 6, true, false, "", 2, 1); err != nil {
t.Fatal(err)
}
db := GetDB()
ctx := context.Background()
fmt.Println(
db.msgDocDatabase.(*unrelation.MsgMongoDriver).GetMsgBySeqIndexIn1Doc(ctx, "100", "si_100_101:0", []int64{1}),
)
//res, err := db.msgDocDatabase.GetMsgBySeqIndexIn1Doc(ctx, "123456", "test:0", []int64{1, 2, 3})
//if err != nil {
// t.Fatal(err)
//}
//db.GetMsgBySeqs(ctx, "100", "si_100_101:0", []int64{6})
//data, _ := json.Marshal(res)
//fmt.Println(string(data))
}
//func Test_Delete(t *testing.T) {
// db := GetDB()
// ctx := context.Background()
// var arr []any
// for i := 0; i < 123; i++ {
// arr = append(arr, []string{"uid_1", "uid_2"})
// }
// if err := db.BatchInsertBlock(ctx, "test", arr, updateKeyDel, 210); err != nil {
// t.Fatal(err)
// }
//}
func TestName(t *testing.T) {
db := GetDB()
var seqs []int64
for i := int64(1); i <= 50; i++ {
seqs = append(seqs, i)
}
msgs, err := db.getMsgBySeqsRange(context.Background(), "4931176757", "si_3866692501_4931176757", seqs, seqs[0], seqs[len(seqs)-1])
if err != nil {
t.Fatal(err)
}
t.Log(msgs)
}
pkg/common/db/controller/push.go
+2 -2
View File
@@ -25,10 +25,10 @@ type PushDatabase interface {
}
type pushDataBase struct {
cache cache.MsgModel
cache cache.ThirdCache
}
func NewPushDatabase(cache cache.MsgModel) PushDatabase {
func NewPushDatabase(cache cache.ThirdCache) PushDatabase {
return &pushDataBase{cache: cache}
}
pkg/common/db/controller/s3.go
+2 -2
View File
@@ -20,9 +20,9 @@ import (
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cont"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/s3"
"github.com/openimsdk/tools/s3/cont"
"github.com/redis/go-redis/v9"
)
pkg/common/db/controller/third.go
+3 -3
View File
@@ -18,9 +18,9 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/db/pagination"
)
type ThirdDatabase interface {
@@ -34,7 +34,7 @@ type ThirdDatabase interface {
}
type thirdDatabase struct {
cache cache.MsgModel
cache cache.ThirdCache
logdb relation.LogInterface
}
@@ -58,7 +58,7 @@ func (t *thirdDatabase) UploadLogs(ctx context.Context, logs []*relation.LogMode
return t.logdb.Create(ctx, logs)
}
func NewThirdDatabase(cache cache.MsgModel, logdb relation.LogInterface) ThirdDatabase {
func NewThirdDatabase(cache cache.ThirdCache, logdb relation.LogInterface) ThirdDatabase {
return &thirdDatabase{cache: cache, logdb: logdb}
}
pkg/common/db/controller/user.go
+21 -34
View File
@@ -16,16 +16,16 @@ package controller
import (
"context"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/db/tx"
"github.com/openimsdk/tools/utils/datautil"
"time"
"github.com/OpenIMSDK/protocol/user"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/protocol/user"
"github.com/openimsdk/tools/errs"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
unrelationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
)
type UserDatabase interface {
@@ -39,13 +39,11 @@ type UserDatabase interface {
FindNotification(ctx context.Context, level int64) (users []*relation.UserModel, err error)
// Create Insert multiple external guarantees that the userID is not repeated and does not exist in the db
Create(ctx context.Context, users []*relation.UserModel) (err error)
// Update update (non-zero value) external guarantee userID exists
//Update(ctx context.Context, user *relation.UserModel) (err error)
// UpdateByMap update (zero value) external guarantee userID exists
UpdateByMap(ctx context.Context, userID string, args map[string]any) (err error)
// FindUser
PageFindUser(ctx context.Context, level1 int64, level2 int64, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error)
//FindUser with keyword
// FindUser with keyword
PageFindUserWithKeyword(ctx context.Context, level1 int64, level2 int64, userID string, nickName string, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error)
// Page If not found, no error is returned
Page(ctx context.Context, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error)
@@ -74,7 +72,7 @@ type UserDatabase interface {
// SetUserStatus Set the user status and store the user status in redis
SetUserStatus(ctx context.Context, userID string, status, platformID int32) error
//CRUD user command
// CRUD user command
AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error
DeleteUserCommand(ctx context.Context, userID string, Type int32, UUID string) error
UpdateUserCommand(ctx context.Context, userID string, Type int32, UUID string, val map[string]any) error
@@ -83,19 +81,19 @@ type UserDatabase interface {
}
type userDatabase struct {
tx tx.CtxTx
tx tx.Tx
userDB relation.UserModelInterface
cache cache.UserCache
mongoDB unrelationtb.UserModelInterface
mongoDB relation.SubscribeUserModelInterface
}
func NewUserDatabase(userDB relation.UserModelInterface, cache cache.UserCache, tx tx.CtxTx, mongoDB unrelationtb.UserModelInterface) UserDatabase {
func NewUserDatabase(userDB relation.UserModelInterface, cache cache.UserCache, tx tx.Tx, mongoDB relation.SubscribeUserModelInterface) UserDatabase {
return &userDatabase{userDB: userDB, cache: cache, tx: tx, mongoDB: mongoDB}
}
func (u *userDatabase) InitOnce(ctx context.Context, users []*relation.UserModel) error {
// Extract user IDs from the given user models.
userIDs := utils.Slice(users, func(e *relation.UserModel) string {
userIDs := datautil.Slice(users, func(e *relation.UserModel) string {
return e.UserID
})
@@ -106,7 +104,7 @@ func (u *userDatabase) InitOnce(ctx context.Context, users []*relation.UserModel
}
// Determine which users are missing from the database.
missingUsers := utils.SliceAnySub(users, existingUsers, func(e *relation.UserModel) string {
missingUsers := datautil.SliceAnySub(users, existingUsers, func(e *relation.UserModel) string {
return e.UserID
})
@@ -127,7 +125,7 @@ func (u *userDatabase) FindWithError(ctx context.Context, userIDs []string) (use
return
}
if len(users) != len(userIDs) {
err = errs.ErrRecordNotFound.Wrap("userID not found")
err = errs.ErrRecordNotFound.WrapMsg("userID not found")
}
return
}
@@ -137,12 +135,10 @@ func (u *userDatabase) Find(ctx context.Context, userIDs []string) (users []*rel
return u.cache.GetUsersInfo(ctx, userIDs)
}
// Find userInfo By Nickname.
func (u *userDatabase) FindByNickname(ctx context.Context, nickname string) (users []*relation.UserModel, err error) {
return u.userDB.TakeByNickname(ctx, nickname)
}
// Find notificationAccouts.
func (u *userDatabase) FindNotification(ctx context.Context, level int64) (users []*relation.UserModel, err error) {
return u.userDB.TakeNotification(ctx, level)
}
@@ -153,20 +149,12 @@ func (u *userDatabase) Create(ctx context.Context, users []*relation.UserModel)
if err = u.userDB.Create(ctx, users); err != nil {
return err
}
return u.cache.DelUsersInfo(utils.Slice(users, func(e *relation.UserModel) string {
return u.cache.DelUsersInfo(datautil.Slice(users, func(e *relation.UserModel) string {
return e.UserID
})...).ExecDel(ctx)
})
}
//// Update (non-zero value) externally guarantees that userID exists.
//func (u *userDatabase) Update(ctx context.Context, user *relation.UserModel) (err error) {
// if err := u.userDB.Update(ctx, user); err != nil {
// return err
// }
// return u.cache.DelUsersInfo(user.UserID).ExecDel(ctx)
//}
// UpdateByMap update (zero value) externally guarantees that userID exists.
func (u *userDatabase) UpdateByMap(ctx context.Context, userID string, args map[string]any) (err error) {
return u.tx.Transaction(ctx, func(ctx context.Context) error {
@@ -186,13 +174,7 @@ func (u *userDatabase) PageFindUser(ctx context.Context, level1 int64, level2 in
return u.userDB.PageFindUser(ctx, level1, level2, pagination)
}
func (u *userDatabase) PageFindUserWithKeyword(
ctx context.Context,
level1 int64,
level2 int64,
userID, nickName string,
pagination pagination.Pagination,
) (count int64, users []*relation.UserModel, err error) {
func (u *userDatabase) PageFindUserWithKeyword(ctx context.Context, level1 int64, level2 int64, userID, nickName string, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error) {
return u.userDB.PageFindUserWithKeyword(ctx, level1, level2, userID, nickName, pagination)
}
@@ -267,19 +249,24 @@ func (u *userDatabase) GetUserStatus(ctx context.Context, userIDs []string) ([]*
func (u *userDatabase) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error {
return u.cache.SetUserStatus(ctx, userID, status, platformID)
}
func (u *userDatabase) AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error {
return u.userDB.AddUserCommand(ctx, userID, Type, UUID, value, ex)
}
func (u *userDatabase) DeleteUserCommand(ctx context.Context, userID string, Type int32, UUID string) error {
return u.userDB.DeleteUserCommand(ctx, userID, Type, UUID)
}
func (u *userDatabase) UpdateUserCommand(ctx context.Context, userID string, Type int32, UUID string, val map[string]any) error {
return u.userDB.UpdateUserCommand(ctx, userID, Type, UUID, val)
}
func (u *userDatabase) GetUserCommands(ctx context.Context, userID string, Type int32) ([]*user.CommandInfoResp, error) {
commands, err := u.userDB.GetUserCommand(ctx, userID, Type)
return commands, err
}
func (u *userDatabase) GetAllUserCommands(ctx context.Context, userID string) ([]*user.AllCommandInfoResp, error) {
commands, err := u.userDB.GetAllUserCommand(ctx, userID)
return commands, err
pkg/common/db/mgo/black.go
+11 -11
View File
@@ -17,9 +17,9 @@ package mgo
import (
"context"
"github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/db/mongoutil"
"github.com/openimsdk/tools/db/pagination"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@@ -63,42 +63,42 @@ func (b *BlackMgo) blacksFilter(blacks []*relation.BlackModel) bson.M {
}
func (b *BlackMgo) Create(ctx context.Context, blacks []*relation.BlackModel) (err error) {
return mgoutil.InsertMany(ctx, b.coll, blacks)
return mongoutil.InsertMany(ctx, b.coll, blacks)
}
func (b *BlackMgo) Delete(ctx context.Context, blacks []*relation.BlackModel) (err error) {
if len(blacks) == 0 {
return nil
}
return mgoutil.DeleteMany(ctx, b.coll, b.blacksFilter(blacks))
return mongoutil.DeleteMany(ctx, b.coll, b.blacksFilter(blacks))
}
func (b *BlackMgo) UpdateByMap(ctx context.Context, ownerUserID, blockUserID string, args map[string]any) (err error) {
if len(args) == 0 {
return nil
}
return mgoutil.UpdateOne(ctx, b.coll, b.blackFilter(ownerUserID, blockUserID), bson.M{"$set": args}, false)
return mongoutil.UpdateOne(ctx, b.coll, b.blackFilter(ownerUserID, blockUserID), bson.M{"$set": args}, false)
}
func (b *BlackMgo) Find(ctx context.Context, blacks []*relation.BlackModel) (blackList []*relation.BlackModel, err error) {
return mgoutil.Find[*relation.BlackModel](ctx, b.coll, b.blacksFilter(blacks))
return mongoutil.Find[*relation.BlackModel](ctx, b.coll, b.blacksFilter(blacks))
}
func (b *BlackMgo) Take(ctx context.Context, ownerUserID, blockUserID string) (black *relation.BlackModel, err error) {
return mgoutil.FindOne[*relation.BlackModel](ctx, b.coll, b.blackFilter(ownerUserID, blockUserID))
return mongoutil.FindOne[*relation.BlackModel](ctx, b.coll, b.blackFilter(ownerUserID, blockUserID))
}
func (b *BlackMgo) FindOwnerBlacks(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (total int64, blacks []*relation.BlackModel, err error) {
return mgoutil.FindPage[*relation.BlackModel](ctx, b.coll, bson.M{"owner_user_id": ownerUserID}, pagination)
return mongoutil.FindPage[*relation.BlackModel](ctx, b.coll, bson.M{"owner_user_id": ownerUserID}, pagination)
}
func (b *BlackMgo) FindOwnerBlackInfos(ctx context.Context, ownerUserID string, userIDs []string) (blacks []*relation.BlackModel, err error) {
if len(userIDs) == 0 {
return mgoutil.Find[*relation.BlackModel](ctx, b.coll, bson.M{"owner_user_id": ownerUserID})
return mongoutil.Find[*relation.BlackModel](ctx, b.coll, bson.M{"owner_user_id": ownerUserID})
}
return mgoutil.Find[*relation.BlackModel](ctx, b.coll, bson.M{"owner_user_id": ownerUserID, "block_user_id": bson.M{"$in": userIDs}})
return mongoutil.Find[*relation.BlackModel](ctx, b.coll, bson.M{"owner_user_id": ownerUserID, "block_user_id": bson.M{"$in": userIDs}})
}
func (b *BlackMgo) FindBlackUserIDs(ctx context.Context, ownerUserID string) (blackUserIDs []string, err error) {
return mgoutil.Find[string](ctx, b.coll, bson.M{"owner_user_id": ownerUserID}, options.Find().SetProjection(bson.M{"_id": 0, "block_user_id": 1}))
return mongoutil.Find[string](ctx, b.coll, bson.M{"owner_user_id": ownerUserID}, options.Find().SetProjection(bson.M{"_id": 0, "block_user_id": 1}))
}
pkg/common/db/mgo/conversation.go
+32 -24
View File
@@ -18,11 +18,11 @@ import (
"context"
"time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/db/mongoutil"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/errs"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@@ -48,15 +48,24 @@ type ConversationMgo struct {
}
func (c *ConversationMgo) Create(ctx context.Context, conversations []*relation.ConversationModel) (err error) {
return mgoutil.InsertMany(ctx, c.coll, conversations)
return mongoutil.InsertMany(ctx, c.coll, conversations)
}
func (c *ConversationMgo) Delete(ctx context.Context, groupIDs []string) (err error) {
return mgoutil.DeleteMany(ctx, c.coll, bson.M{"group_id": bson.M{"$in": groupIDs}})
return mongoutil.DeleteMany(ctx, c.coll, bson.M{"group_id": bson.M{"$in": groupIDs}})
}
func (c *ConversationMgo) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (rows int64, err error) {
res, err := mgoutil.UpdateMany(ctx, c.coll, bson.M{"owner_user_id": bson.M{"$in": userIDs}, "conversation_id": conversationID}, bson.M{"$set": args})
if len(args) == 0 {
return 0, nil
}
filter := bson.M{
"conversation_id": conversationID,
}
if len(userIDs) > 0 {
filter["owner_user_id"] = bson.M{"$in": userIDs}
}
res, err := mongoutil.UpdateMany(ctx, c.coll, filter, bson.M{"$set": args})
if err != nil {
return 0, err
}
@@ -64,36 +73,35 @@ func (c *ConversationMgo) UpdateByMap(ctx context.Context, userIDs []string, con
}
func (c *ConversationMgo) Update(ctx context.Context, conversation *relation.ConversationModel) (err error) {
return mgoutil.UpdateOne(ctx, c.coll, bson.M{"owner_user_id": conversation.OwnerUserID, "conversation_id": conversation.ConversationID}, bson.M{"$set": conversation}, true)
return mongoutil.UpdateOne(ctx, c.coll, bson.M{"owner_user_id": conversation.OwnerUserID, "conversation_id": conversation.ConversationID}, bson.M{"$set": conversation}, true)
}
func (c *ConversationMgo) Find(ctx context.Context, ownerUserID string, conversationIDs []string) (conversations []*relation.ConversationModel, err error) {
return mgoutil.Find[*relation.ConversationModel](ctx, c.coll, bson.M{"owner_user_id": ownerUserID, "conversation_id": bson.M{"$in": conversationIDs}})
return mongoutil.Find[*relation.ConversationModel](ctx, c.coll, bson.M{"owner_user_id": ownerUserID, "conversation_id": bson.M{"$in": conversationIDs}})
}
func (c *ConversationMgo) FindUserID(ctx context.Context, userIDs []string, conversationIDs []string) ([]string, error) {
return mgoutil.Find[string](
return mongoutil.Find[string](
ctx,
c.coll,
bson.M{"owner_user_id": bson.M{"$in": userIDs}, "conversation_id": bson.M{"$in": conversationIDs}},
options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}),
)
}
func (c *ConversationMgo) FindUserIDAllConversationID(ctx context.Context, userID string) ([]string, error) {
return mgoutil.Find[string](ctx, c.coll, bson.M{"owner_user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "conversation_id": 1}))
return mongoutil.Find[string](ctx, c.coll, bson.M{"owner_user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "conversation_id": 1}))
}
func (c *ConversationMgo) Take(ctx context.Context, userID, conversationID string) (conversation *relation.ConversationModel, err error) {
return mgoutil.FindOne[*relation.ConversationModel](ctx, c.coll, bson.M{"owner_user_id": userID, "conversation_id": conversationID})
return mongoutil.FindOne[*relation.ConversationModel](ctx, c.coll, bson.M{"owner_user_id": userID, "conversation_id": conversationID})
}
func (c *ConversationMgo) FindConversationID(ctx context.Context, userID string, conversationIDs []string) (existConversationID []string, err error) {
return mgoutil.Find[string](ctx, c.coll, bson.M{"owner_user_id": userID, "conversation_id": bson.M{"$in": conversationIDs}}, options.Find().SetProjection(bson.M{"_id": 0, "conversation_id": 1}))
return mongoutil.Find[string](ctx, c.coll, bson.M{"owner_user_id": userID, "conversation_id": bson.M{"$in": conversationIDs}}, options.Find().SetProjection(bson.M{"_id": 0, "conversation_id": 1}))
}
func (c *ConversationMgo) FindUserIDAllConversations(ctx context.Context, userID string) (conversations []*relation.ConversationModel, err error) {
return mgoutil.Find[*relation.ConversationModel](ctx, c.coll, bson.M{"owner_user_id": userID})
return mongoutil.Find[*relation.ConversationModel](ctx, c.coll, bson.M{"owner_user_id": userID})
}
func (c *ConversationMgo) FindRecvMsgUserIDs(ctx context.Context, conversationID string, recvOpts []int) ([]string, error) {
@@ -103,22 +111,22 @@ func (c *ConversationMgo) FindRecvMsgUserIDs(ctx context.Context, conversationID
} else {
filter = bson.M{"conversation_id": conversationID, "recv_msg_opt": bson.M{"$in": recvOpts}}
}
return mgoutil.Find[string](ctx, c.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}))
return mongoutil.Find[string](ctx, c.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}))
}
func (c *ConversationMgo) GetUserRecvMsgOpt(ctx context.Context, ownerUserID, conversationID string) (opt int, err error) {
return mgoutil.FindOne[int](ctx, c.coll, bson.M{"owner_user_id": ownerUserID, "conversation_id": conversationID}, options.FindOne().SetProjection(bson.M{"recv_msg_opt": 1}))
return mongoutil.FindOne[int](ctx, c.coll, bson.M{"owner_user_id": ownerUserID, "conversation_id": conversationID}, options.FindOne().SetProjection(bson.M{"recv_msg_opt": 1}))
}
func (c *ConversationMgo) GetAllConversationIDs(ctx context.Context) ([]string, error) {
return mgoutil.Aggregate[string](ctx, c.coll, []bson.M{
return mongoutil.Aggregate[string](ctx, c.coll, []bson.M{
{"$group": bson.M{"_id": "$conversation_id"}},
{"$project": bson.M{"_id": 0, "conversation_id": "$_id"}},
})
}
func (c *ConversationMgo) GetAllConversationIDsNumber(ctx context.Context) (int64, error) {
counts, err := mgoutil.Aggregate[int64](ctx, c.coll, []bson.M{
counts, err := mongoutil.Aggregate[int64](ctx, c.coll, []bson.M{
{"$group": bson.M{"_id": "$conversation_id"}},
{"$group": bson.M{"_id": nil, "count": bson.M{"$sum": 1}}},
{"$project": bson.M{"_id": 0}},
@@ -133,16 +141,16 @@ func (c *ConversationMgo) GetAllConversationIDsNumber(ctx context.Context) (int6
}
func (c *ConversationMgo) PageConversationIDs(ctx context.Context, pagination pagination.Pagination) (conversationIDs []string, err error) {
return mgoutil.FindPageOnly[string](ctx, c.coll, bson.M{}, pagination, options.Find().SetProjection(bson.M{"conversation_id": 1}))
return mongoutil.FindPageOnly[string](ctx, c.coll, bson.M{}, pagination, options.Find().SetProjection(bson.M{"conversation_id": 1}))
}
func (c *ConversationMgo) GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*relation.ConversationModel, error) {
return mgoutil.Find[*relation.ConversationModel](ctx, c.coll, bson.M{"conversation_id": bson.M{"$in": conversationIDs}})
return mongoutil.Find[*relation.ConversationModel](ctx, c.coll, bson.M{"conversation_id": bson.M{"$in": conversationIDs}})
}
func (c *ConversationMgo) GetConversationIDsNeedDestruct(ctx context.Context) ([]*relation.ConversationModel, error) {
//"is_msg_destruct = 1 && msg_destruct_time != 0 && (UNIX_TIMESTAMP(NOW()) > (msg_destruct_time + UNIX_TIMESTAMP(latest_msg_destruct_time)) || latest_msg_destruct_time is NULL)"
return mgoutil.Find[*relation.ConversationModel](ctx, c.coll, bson.M{
// "is_msg_destruct = 1 && msg_destruct_time != 0 && (UNIX_TIMESTAMP(NOW()) > (msg_destruct_time + UNIX_TIMESTAMP(latest_msg_destruct_time)) || latest_msg_destruct_time is NULL)"
return mongoutil.Find[*relation.ConversationModel](ctx, c.coll, bson.M{
"is_msg_destruct": 1,
"msg_destruct_time": bson.M{"$ne": 0},
"$or": []bson.M{
@@ -162,7 +170,7 @@ func (c *ConversationMgo) GetConversationIDsNeedDestruct(ctx context.Context) ([
}
func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) {
return mgoutil.Find[string](
return mongoutil.Find[string](
ctx,
c.coll,
bson.M{"conversation_id": conversationID, "recv_msg_opt": bson.M{"$ne": constant.ReceiveMessage}},
pkg/common/db/s3/doc.go → pkg/common/db/mgo/doc.go
+2 -2
View File
@@ -1,4 +1,4 @@
// Copyright © 2023 OpenIM. All rights reserved.
// Copyright © 2024 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,4 +12,4 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package s3 // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
package mgo // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
pkg/common/db/mgo/friend.go
+13 -13
View File
@@ -17,9 +17,9 @@ package mgo
import (
"context"
"github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/db/mongoutil"
"github.com/openimsdk/tools/db/pagination"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@@ -48,7 +48,7 @@ func NewFriendMongo(db *mongo.Database) (relation.FriendModelInterface, error) {
// Create inserts multiple friend records.
func (f *FriendMgo) Create(ctx context.Context, friends []*relation.FriendModel) error {
return mgoutil.InsertMany(ctx, f.coll, friends)
return mongoutil.InsertMany(ctx, f.coll, friends)
}
// Delete removes specified friends of the owner user.
@@ -57,7 +57,7 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID
"owner_user_id": ownerUserID,
"friend_user_id": bson.M{"$in": friendUserIDs},
}
return mgoutil.DeleteOne(ctx, f.coll, filter)
return mongoutil.DeleteOne(ctx, f.coll, filter)
}
// UpdateByMap updates specific fields of a friend document using a map.
@@ -69,7 +69,7 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU
"owner_user_id": ownerUserID,
"friend_user_id": friendUserID,
}
return mgoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true)
return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true)
}
// Update modifies multiple friend documents.
@@ -92,7 +92,7 @@ func (f *FriendMgo) Take(ctx context.Context, ownerUserID, friendUserID string)
"owner_user_id": ownerUserID,
"friend_user_id": friendUserID,
}
return mgoutil.FindOne[*relation.FriendModel](ctx, f.coll, filter)
return mongoutil.FindOne[*relation.FriendModel](ctx, f.coll, filter)
}
// FindUserState finds the friendship status between two users.
@@ -103,7 +103,7 @@ func (f *FriendMgo) FindUserState(ctx context.Context, userID1, userID2 string)
{"owner_user_id": userID2, "friend_user_id": userID1},
},
}
return mgoutil.Find[*relation.FriendModel](ctx, f.coll, filter)
return mongoutil.Find[*relation.FriendModel](ctx, f.coll, filter)
}
// FindFriends retrieves a list of friends for a given owner. Missing friends do not cause an error.
@@ -112,7 +112,7 @@ func (f *FriendMgo) FindFriends(ctx context.Context, ownerUserID string, friendU
"owner_user_id": ownerUserID,
"friend_user_id": bson.M{"$in": friendUserIDs},
}
return mgoutil.Find[*relation.FriendModel](ctx, f.coll, filter)
return mongoutil.Find[*relation.FriendModel](ctx, f.coll, filter)
}
// FindReversalFriends finds users who have added the specified user as a friend.
@@ -121,25 +121,25 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string
"owner_user_id": bson.M{"$in": ownerUserIDs},
"friend_user_id": friendUserID,
}
return mgoutil.Find[*relation.FriendModel](ctx, f.coll, filter)
return mongoutil.Find[*relation.FriendModel](ctx, f.coll, filter)
}
// FindOwnerFriends retrieves a paginated list of friends for a given owner.
func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) {
filter := bson.M{"owner_user_id": ownerUserID}
return mgoutil.FindPage[*relation.FriendModel](ctx, f.coll, filter, pagination)
return mongoutil.FindPage[*relation.FriendModel](ctx, f.coll, filter, pagination)
}
// FindInWhoseFriends finds users who have added the specified user as a friend, with pagination.
func (f *FriendMgo) FindInWhoseFriends(ctx context.Context, friendUserID string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) {
filter := bson.M{"friend_user_id": friendUserID}
return mgoutil.FindPage[*relation.FriendModel](ctx, f.coll, filter, pagination)
return mongoutil.FindPage[*relation.FriendModel](ctx, f.coll, filter, pagination)
}
// FindFriendUserIDs retrieves a list of friend user IDs for a given owner.
func (f *FriendMgo) FindFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) {
filter := bson.M{"owner_user_id": ownerUserID}
return mgoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}))
return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}))
}
func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) error {
@@ -158,6 +158,6 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien
update := bson.M{"$set": val}
// Perform the update operation for all matching documents
_, err := mgoutil.UpdateMany(ctx, f.coll, filter, update)
_, err := mongoutil.UpdateMany(ctx, f.coll, filter, update)
return err
}
pkg/common/db/mgo/friend_request.go
+10 -10
View File
@@ -17,9 +17,9 @@ package mgo
import (
"context"
"github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/db/mongoutil"
"github.com/openimsdk/tools/db/pagination"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@@ -45,11 +45,11 @@ type FriendRequestMgo struct {
}
func (f *FriendRequestMgo) FindToUserID(ctx context.Context, toUserID string, pagination pagination.Pagination) (total int64, friendRequests []*relation.FriendRequestModel, err error) {
return mgoutil.FindPage[*relation.FriendRequestModel](ctx, f.coll, bson.M{"to_user_id": toUserID}, pagination)
return mongoutil.FindPage[*relation.FriendRequestModel](ctx, f.coll, bson.M{"to_user_id": toUserID}, pagination)
}
func (f *FriendRequestMgo) FindFromUserID(ctx context.Context, fromUserID string, pagination pagination.Pagination) (total int64, friendRequests []*relation.FriendRequestModel, err error) {
return mgoutil.FindPage[*relation.FriendRequestModel](ctx, f.coll, bson.M{"from_user_id": fromUserID}, pagination)
return mongoutil.FindPage[*relation.FriendRequestModel](ctx, f.coll, bson.M{"from_user_id": fromUserID}, pagination)
}
func (f *FriendRequestMgo) FindBothFriendRequests(ctx context.Context, fromUserID, toUserID string) (friends []*relation.FriendRequestModel, err error) {
@@ -57,22 +57,22 @@ func (f *FriendRequestMgo) FindBothFriendRequests(ctx context.Context, fromUserI
{"from_user_id": fromUserID, "to_user_id": toUserID},
{"from_user_id": toUserID, "to_user_id": fromUserID},
}}
return mgoutil.Find[*relation.FriendRequestModel](ctx, f.coll, filter)
return mongoutil.Find[*relation.FriendRequestModel](ctx, f.coll, filter)
}
func (f *FriendRequestMgo) Create(ctx context.Context, friendRequests []*relation.FriendRequestModel) error {
return mgoutil.InsertMany(ctx, f.coll, friendRequests)
return mongoutil.InsertMany(ctx, f.coll, friendRequests)
}
func (f *FriendRequestMgo) Delete(ctx context.Context, fromUserID, toUserID string) (err error) {
return mgoutil.DeleteOne(ctx, f.coll, bson.M{"from_user_id": fromUserID, "to_user_id": toUserID})
return mongoutil.DeleteOne(ctx, f.coll, bson.M{"from_user_id": fromUserID, "to_user_id": toUserID})
}
func (f *FriendRequestMgo) UpdateByMap(ctx context.Context, formUserID, toUserID string, args map[string]any) (err error) {
if len(args) == 0 {
return nil
}
return mgoutil.UpdateOne(ctx, f.coll, bson.M{"from_user_id": formUserID, "to_user_id": toUserID}, bson.M{"$set": args}, true)
return mongoutil.UpdateOne(ctx, f.coll, bson.M{"from_user_id": formUserID, "to_user_id": toUserID}, bson.M{"$set": args}, true)
}
func (f *FriendRequestMgo) Update(ctx context.Context, friendRequest *relation.FriendRequestModel) (err error) {
@@ -99,11 +99,11 @@ func (f *FriendRequestMgo) Update(ctx context.Context, friendRequest *relation.F
return nil
}
filter := bson.M{"from_user_id": friendRequest.FromUserID, "to_user_id": friendRequest.ToUserID}
return mgoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": updater}, true)
return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": updater}, true)
}
func (f *FriendRequestMgo) Find(ctx context.Context, fromUserID, toUserID string) (friendRequest *relation.FriendRequestModel, err error) {
return mgoutil.FindOne[*relation.FriendRequestModel](ctx, f.coll, bson.M{"from_user_id": fromUserID, "to_user_id": toUserID})
return mongoutil.FindOne[*relation.FriendRequestModel](ctx, f.coll, bson.M{"from_user_id": fromUserID, "to_user_id": toUserID})
}
func (f *FriendRequestMgo) Take(ctx context.Context, fromUserID, toUserID string) (friendRequest *relation.FriendRequestModel, err error) {
pkg/common/db/mgo/group.go
+14 -12
View File
@@ -18,11 +18,11 @@ import (
"context"
"time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/db/mongoutil"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/errs"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@@ -47,7 +47,7 @@ type GroupMgo struct {
}
func (g *GroupMgo) Create(ctx context.Context, groups []*relation.GroupModel) (err error) {
return mgoutil.InsertMany(ctx, g.coll, groups)
return mongoutil.InsertMany(ctx, g.coll, groups)
}
func (g *GroupMgo) UpdateStatus(ctx context.Context, groupID string, status int32) (err error) {
@@ -58,21 +58,23 @@ func (g *GroupMgo) UpdateMap(ctx context.Context, groupID string, args map[strin
if len(args) == 0 {
return nil
}
return mgoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID}, bson.M{"$set": args}, true)
return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID}, bson.M{"$set": args}, true)
}
func (g *GroupMgo) Find(ctx context.Context, groupIDs []string) (groups []*relation.GroupModel, err error) {
return mgoutil.Find[*relation.GroupModel](ctx, g.coll, bson.M{"group_id": bson.M{"$in": groupIDs}})
return mongoutil.Find[*relation.GroupModel](ctx, g.coll, bson.M{"group_id": bson.M{"$in": groupIDs}})
}
func (g *GroupMgo) Take(ctx context.Context, groupID string) (group *relation.GroupModel, err error) {
return mgoutil.FindOne[*relation.GroupModel](ctx, g.coll, bson.M{"group_id": groupID})
return mongoutil.FindOne[*relation.GroupModel](ctx, g.coll, bson.M{"group_id": groupID})
}
func (g *GroupMgo) Search(ctx context.Context, keyword string, pagination pagination.Pagination) (total int64, groups []*relation.GroupModel, err error) {
// Define the sorting options
opts := options.Find().SetSort(bson.D{{Key: "created_at", Value: -1}})
return mgoutil.FindPage[*relation.GroupModel](ctx, g.coll, bson.M{
// Perform the search with pagination and sorting
return mongoutil.FindPage[*relation.GroupModel](ctx, g.coll, bson.M{
"group_name": bson.M{"$regex": keyword},
"status": bson.M{"$ne": constant.GroupStatusDismissed},
}, pagination, opts)
@@ -80,9 +82,9 @@ func (g *GroupMgo) Search(ctx context.Context, keyword string, pagination pagina
func (g *GroupMgo) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) {
if before == nil {
return mgoutil.Count(ctx, g.coll, bson.M{})
return mongoutil.Count(ctx, g.coll, bson.M{})
}
return mgoutil.Count(ctx, g.coll, bson.M{"create_time": bson.M{"$lt": before}})
return mongoutil.Count(ctx, g.coll, bson.M{"create_time": bson.M{"$lt": before}})
}
func (g *GroupMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) {
@@ -113,7 +115,7 @@ func (g *GroupMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time,
Date string `bson:"_id"`
Count int64 `bson:"count"`
}
items, err := mgoutil.Aggregate[Item](ctx, g.coll, pipeline)
items, err := mongoutil.Aggregate[Item](ctx, g.coll, pipeline)
if err != nil {
return nil, err
}
pkg/common/db/mgo/group_member.go
+16 -16
View File
@@ -17,11 +17,11 @@ package mgo
import (
"context"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/tools/db/mongoutil"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/errs"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@@ -47,7 +47,7 @@ type GroupMemberMgo struct {
}
func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*relation.GroupMemberModel) (err error) {
return mgoutil.InsertMany(ctx, g.coll, groupMembers)
return mongoutil.InsertMany(ctx, g.coll, groupMembers)
}
func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []string) (err error) {
@@ -55,7 +55,7 @@ func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []s
if len(userIDs) > 0 {
filter["user_id"] = bson.M{"$in": userIDs}
}
return mgoutil.DeleteMany(ctx, g.coll, filter)
return mongoutil.DeleteMany(ctx, g.coll, filter)
}
func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, userID string, roleLevel int32) error {
@@ -63,41 +63,41 @@ func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, us
}
func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID string, data map[string]any) (err error) {
return mgoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true)
return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true)
}
func (g *GroupMemberMgo) Find(ctx context.Context, groupIDs []string, userIDs []string, roleLevels []int32) (groupMembers []*relation.GroupMemberModel, err error) {
//TODO implement me
// TODO implement me
panic("implement me")
}
func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) {
return mgoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}))
return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}))
}
func (g *GroupMemberMgo) Take(ctx context.Context, groupID string, userID string) (groupMember *relation.GroupMemberModel, err error) {
return mgoutil.FindOne[*relation.GroupMemberModel](ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID})
return mongoutil.FindOne[*relation.GroupMemberModel](ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID})
}
func (g *GroupMemberMgo) TakeOwner(ctx context.Context, groupID string) (groupMember *relation.GroupMemberModel, err error) {
return mgoutil.FindOne[*relation.GroupMemberModel](ctx, g.coll, bson.M{"group_id": groupID, "role_level": constant.GroupOwner})
return mongoutil.FindOne[*relation.GroupMemberModel](ctx, g.coll, bson.M{"group_id": groupID, "role_level": constant.GroupOwner})
}
func (g *GroupMemberMgo) FindRoleLevelUserIDs(ctx context.Context, groupID string, roleLevel int32) ([]string, error) {
return mgoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID, "role_level": roleLevel}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}))
return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID, "role_level": roleLevel}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}))
}
func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*relation.GroupMemberModel, err error) {
filter := bson.M{"group_id": groupID, "nickname": bson.M{"$regex": keyword}}
return mgoutil.FindPage[*relation.GroupMemberModel](ctx, g.coll, filter, pagination)
return mongoutil.FindPage[*relation.GroupMemberModel](ctx, g.coll, filter, pagination)
}
func (g *GroupMemberMgo) FindUserJoinedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) {
return mgoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}))
return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}))
}
func (g *GroupMemberMgo) TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) {
return mgoutil.Count(ctx, g.coll, bson.M{"group_id": groupID})
return mongoutil.Count(ctx, g.coll, bson.M{"group_id": groupID})
}
func (g *GroupMemberMgo) FindUserManagedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) {
@@ -107,7 +107,7 @@ func (g *GroupMemberMgo) FindUserManagedGroupID(ctx context.Context, userID stri
"$in": []int{constant.GroupOwner, constant.GroupAdmin},
},
}
return mgoutil.Find[string](ctx, g.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}))
return mongoutil.Find[string](ctx, g.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}))
}
func (g *GroupMemberMgo) IsUpdateRoleLevel(data map[string]any) bool {
pkg/common/db/mgo/group_request.go
+10 -10
View File
@@ -17,10 +17,10 @@ package mgo
import (
"context"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/db/mongoutil"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/errs"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@@ -46,29 +46,29 @@ type GroupRequestMgo struct {
}
func (g *GroupRequestMgo) Create(ctx context.Context, groupRequests []*relation.GroupRequestModel) (err error) {
return mgoutil.InsertMany(ctx, g.coll, groupRequests)
return mongoutil.InsertMany(ctx, g.coll, groupRequests)
}
func (g *GroupRequestMgo) Delete(ctx context.Context, groupID string, userID string) (err error) {
return mgoutil.DeleteOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID})
return mongoutil.DeleteOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID})
}
func (g *GroupRequestMgo) UpdateHandler(ctx context.Context, groupID string, userID string, handledMsg string, handleResult int32) (err error) {
return mgoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": bson.M{"handle_msg": handledMsg, "handle_result": handleResult}}, true)
return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": bson.M{"handle_msg": handledMsg, "handle_result": handleResult}}, true)
}
func (g *GroupRequestMgo) Take(ctx context.Context, groupID string, userID string) (groupRequest *relation.GroupRequestModel, err error) {
return mgoutil.FindOne[*relation.GroupRequestModel](ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID})
return mongoutil.FindOne[*relation.GroupRequestModel](ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID})
}
func (g *GroupRequestMgo) FindGroupRequests(ctx context.Context, groupID string, userIDs []string) ([]*relation.GroupRequestModel, error) {
return mgoutil.Find[*relation.GroupRequestModel](ctx, g.coll, bson.M{"group_id": groupID, "user_id": bson.M{"$in": userIDs}})
return mongoutil.Find[*relation.GroupRequestModel](ctx, g.coll, bson.M{"group_id": groupID, "user_id": bson.M{"$in": userIDs}})
}
func (g *GroupRequestMgo) Page(ctx context.Context, userID string, pagination pagination.Pagination) (total int64, groups []*relation.GroupRequestModel, err error) {
return mgoutil.FindPage[*relation.GroupRequestModel](ctx, g.coll, bson.M{"user_id": userID}, pagination)
return mongoutil.FindPage[*relation.GroupRequestModel](ctx, g.coll, bson.M{"user_id": userID}, pagination)
}
func (g *GroupRequestMgo) PageGroup(ctx context.Context, groupIDs []string, pagination pagination.Pagination) (total int64, groups []*relation.GroupRequestModel, err error) {
return mgoutil.FindPage[*relation.GroupRequestModel](ctx, g.coll, bson.M{"group_id": bson.M{"$in": groupIDs}}, pagination)
return mongoutil.FindPage[*relation.GroupRequestModel](ctx, g.coll, bson.M{"group_id": bson.M{"$in": groupIDs}}, pagination)
}
pkg/common/db/mgo/log.go
+8 -8
View File
@@ -18,9 +18,9 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/db/mongoutil"
"github.com/openimsdk/tools/db/pagination"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@@ -57,7 +57,7 @@ type LogMgo struct {
}
func (l *LogMgo) Create(ctx context.Context, log []*relation.LogModel) error {
return mgoutil.InsertMany(ctx, l.coll, log)
return mongoutil.InsertMany(ctx, l.coll, log)
}
func (l *LogMgo) Search(ctx context.Context, keyword string, start time.Time, end time.Time, pagination pagination.Pagination) (int64, []*relation.LogModel, error) {
@@ -65,19 +65,19 @@ func (l *LogMgo) Search(ctx context.Context, keyword string, start time.Time, en
if keyword != "" {
filter["user_id"] = bson.M{"$regex": keyword}
}
return mgoutil.FindPage[*relation.LogModel](ctx, l.coll, filter, pagination, options.Find().SetSort(bson.M{"create_time": -1}))
return mongoutil.FindPage[*relation.LogModel](ctx, l.coll, filter, pagination, options.Find().SetSort(bson.M{"create_time": -1}))
}
func (l *LogMgo) Delete(ctx context.Context, logID []string, userID string) error {
if userID == "" {
return mgoutil.DeleteMany(ctx, l.coll, bson.M{"log_id": bson.M{"$in": logID}})
return mongoutil.DeleteMany(ctx, l.coll, bson.M{"log_id": bson.M{"$in": logID}})
}
return mgoutil.DeleteMany(ctx, l.coll, bson.M{"log_id": bson.M{"$in": logID}, "user_id": userID})
return mongoutil.DeleteMany(ctx, l.coll, bson.M{"log_id": bson.M{"$in": logID}, "user_id": userID})
}
func (l *LogMgo) Get(ctx context.Context, logIDs []string, userID string) ([]*relation.LogModel, error) {
if userID == "" {
return mgoutil.Find[*relation.LogModel](ctx, l.coll, bson.M{"log_id": bson.M{"$in": logIDs}})
return mongoutil.Find[*relation.LogModel](ctx, l.coll, bson.M{"log_id": bson.M{"$in": logIDs}})
}
return mgoutil.Find[*relation.LogModel](ctx, l.coll, bson.M{"log_id": bson.M{"$in": logIDs}, "user_id": userID})
return mongoutil.Find[*relation.LogModel](ctx, l.coll, bson.M{"log_id": bson.M{"$in": logIDs}, "user_id": userID})
}
pkg/common/db/unrelation/msg.go → pkg/common/db/mgo/msg.go
+353 -529
View File
File diff suppressed because it is too large Load Diff
pkg/common/db/mgo/object.go
+6 -6
View File
@@ -17,9 +17,9 @@ package mgo
import (
"context"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/mgoutil"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/db/mongoutil"
"github.com/openimsdk/tools/errs"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@@ -54,16 +54,16 @@ func (o *S3Mongo) SetObject(ctx context.Context, obj *relation.ObjectModel) erro
"group": obj.Group,
"create_time": obj.CreateTime,
}
return mgoutil.UpdateOne(ctx, o.coll, filter, bson.M{"$set": update}, false, options.Update().SetUpsert(true))
return mongoutil.UpdateOne(ctx, o.coll, filter, bson.M{"$set": update}, false, options.Update().SetUpsert(true))
}
func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*relation.ObjectModel, error) {
if engine == "" {
return mgoutil.FindOne[*relation.ObjectModel](ctx, o.coll, bson.M{"name": name})
return mongoutil.FindOne[*relation.ObjectModel](ctx, o.coll, bson.M{"name": name})
}
return mgoutil.FindOne[*relation.ObjectModel](ctx, o.coll, bson.M{"name": name, "engine": engine})
return mongoutil.FindOne[*relation.ObjectModel](ctx, o.coll, bson.M{"name": name, "engine": engine})
}
func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error {
return mgoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine})
return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine})
}
pkg/common/db/unrelation/user.go → pkg/common/db/mgo/subscribe.go
+8 -8
View File
@@ -12,13 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package unrelation
package mgo
import (
"context"
"github.com/OpenIMSDK/tools/errs"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/tools/errs"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@@ -35,9 +35,9 @@ const (
MaximumSubscription = 3000
)
func NewUserMongoDriver(database *mongo.Database) unrelation.UserModelInterface {
func NewUserMongoDriver(database *mongo.Database) relation.SubscribeUserModelInterface {
return &UserMongoDriver{
userCollection: database.Collection(unrelation.SubscribeUser),
userCollection: database.Collection(relation.SubscribeUser),
}
}
@@ -117,7 +117,7 @@ func (u *UserMongoDriver) AddSubscriptionList(ctx context.Context, userID string
opts,
)
if err != nil {
return errs.Wrap(err, "transaction failed")
return errs.WrapMsg(err, "transaction failed")
}
}
return nil
@@ -155,7 +155,7 @@ func (u *UserMongoDriver) RemoveSubscribedListFromUser(ctx context.Context, user
// GetAllSubscribeList Get all users subscribed by this user.
func (u *UserMongoDriver) GetAllSubscribeList(ctx context.Context, userID string) (userIDList []string, err error) {
var user unrelation.UserModel
var user relation.SubscribeUserModel
cursor := u.userCollection.FindOne(
ctx,
bson.M{"user_id": SubscriptionPrefix + userID})
@@ -172,7 +172,7 @@ func (u *UserMongoDriver) GetAllSubscribeList(ctx context.Context, userID string
// GetSubscribedList Get the user subscribed by those users.
func (u *UserMongoDriver) GetSubscribedList(ctx context.Context, userID string) (userIDList []string, err error) {
var user unrelation.UserModel
var user relation.SubscribeUserModel
cursor := u.userCollection.FindOne(
ctx,
bson.M{"user_id": SubscribedPrefix + userID})
pkg/common/db/mgo/user.go
+19 -19
View File
@@ -18,11 +18,11 @@ import (
"context"
"time"
"github.com/OpenIMSDK/protocol/user"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/protocol/user"
"github.com/openimsdk/tools/db/mongoutil"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/errs"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
@@ -48,34 +48,34 @@ type UserMgo struct {
}
func (u *UserMgo) Create(ctx context.Context, users []*relation.UserModel) error {
return mgoutil.InsertMany(ctx, u.coll, users)
return mongoutil.InsertMany(ctx, u.coll, users)
}
func (u *UserMgo) UpdateByMap(ctx context.Context, userID string, args map[string]any) (err error) {
if len(args) == 0 {
return nil
}
return mgoutil.UpdateOne(ctx, u.coll, bson.M{"user_id": userID}, bson.M{"$set": args}, true)
return mongoutil.UpdateOne(ctx, u.coll, bson.M{"user_id": userID}, bson.M{"$set": args}, true)
}
func (u *UserMgo) Find(ctx context.Context, userIDs []string) (users []*relation.UserModel, err error) {
return mgoutil.Find[*relation.UserModel](ctx, u.coll, bson.M{"user_id": bson.M{"$in": userIDs}})
return mongoutil.Find[*relation.UserModel](ctx, u.coll, bson.M{"user_id": bson.M{"$in": userIDs}})
}
func (u *UserMgo) Take(ctx context.Context, userID string) (user *relation.UserModel, err error) {
return mgoutil.FindOne[*relation.UserModel](ctx, u.coll, bson.M{"user_id": userID})
return mongoutil.FindOne[*relation.UserModel](ctx, u.coll, bson.M{"user_id": userID})
}
func (u *UserMgo) TakeNotification(ctx context.Context, level int64) (user []*relation.UserModel, err error) {
return mgoutil.Find[*relation.UserModel](ctx, u.coll, bson.M{"app_manger_level": level})
return mongoutil.Find[*relation.UserModel](ctx, u.coll, bson.M{"app_manger_level": level})
}
func (u *UserMgo) TakeByNickname(ctx context.Context, nickname string) (user []*relation.UserModel, err error) {
return mgoutil.Find[*relation.UserModel](ctx, u.coll, bson.M{"nickname": nickname})
return mongoutil.Find[*relation.UserModel](ctx, u.coll, bson.M{"nickname": nickname})
}
func (u *UserMgo) Page(ctx context.Context, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error) {
return mgoutil.FindPage[*relation.UserModel](ctx, u.coll, bson.M{}, pagination)
return mongoutil.FindPage[*relation.UserModel](ctx, u.coll, bson.M{}, pagination)
}
func (u *UserMgo) PageFindUser(ctx context.Context, level1 int64, level2 int64, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error) {
@@ -86,7 +86,7 @@ func (u *UserMgo) PageFindUser(ctx context.Context, level1 int64, level2 int64,
},
}
return mgoutil.FindPage[*relation.UserModel](ctx, u.coll, query, pagination)
return mongoutil.FindPage[*relation.UserModel](ctx, u.coll, query, pagination)
}
func (u *UserMgo) PageFindUserWithKeyword(
@@ -121,26 +121,26 @@ func (u *UserMgo) PageFindUserWithKeyword(
}
// Perform the paginated search
return mgoutil.FindPage[*relation.UserModel](ctx, u.coll, query, pagination)
return mongoutil.FindPage[*relation.UserModel](ctx, u.coll, query, pagination)
}
func (u *UserMgo) GetAllUserID(ctx context.Context, pagination pagination.Pagination) (int64, []string, error) {
return mgoutil.FindPage[string](ctx, u.coll, bson.M{}, pagination, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}))
return mongoutil.FindPage[string](ctx, u.coll, bson.M{}, pagination, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}))
}
func (u *UserMgo) Exist(ctx context.Context, userID string) (exist bool, err error) {
return mgoutil.Exist(ctx, u.coll, bson.M{"user_id": userID})
return mongoutil.Exist(ctx, u.coll, bson.M{"user_id": userID})
}
func (u *UserMgo) GetUserGlobalRecvMsgOpt(ctx context.Context, userID string) (opt int, err error) {
return mgoutil.FindOne[int](ctx, u.coll, bson.M{"user_id": userID}, options.FindOne().SetProjection(bson.M{"_id": 0, "global_recv_msg_opt": 1}))
return mongoutil.FindOne[int](ctx, u.coll, bson.M{"user_id": userID}, options.FindOne().SetProjection(bson.M{"_id": 0, "global_recv_msg_opt": 1}))
}
func (u *UserMgo) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) {
if before == nil {
return mgoutil.Count(ctx, u.coll, bson.M{})
return mongoutil.Count(ctx, u.coll, bson.M{})
}
return mgoutil.Count(ctx, u.coll, bson.M{"create_time": bson.M{"$lt": before}})
return mongoutil.Count(ctx, u.coll, bson.M{"create_time": bson.M{"$lt": before}})
}
func (u *UserMgo) AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error {
@@ -308,7 +308,7 @@ func (u *UserMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time,
Date string `bson:"_id"`
Count int64 `bson:"count"`
}
items, err := mgoutil.Aggregate[Item](ctx, u.coll, pipeline)
items, err := mongoutil.Aggregate[Item](ctx, u.coll, pipeline)
if err != nil {
return nil, err
}
pkg/common/db/s3/cont/consts.go
-38
View File
@@ -1,38 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cont
const (
// hashPath defines the storage path for hash data within the 'openim' directory.
hashPath = "openim/data/hash/"
// tempPath specifies the directory for temporary files in the 'openim' structure.
tempPath = "openim/temp/"
// DirectPath indicates the directory for direct uploads or access within the 'openim' structure.
DirectPath = "openim/direct"
// UploadTypeMultipart represents the identifier for multipart uploads,
// allowing large files to be uploaded in chunks.
UploadTypeMultipart = 1
// UploadTypePresigned signifies the use of presigned URLs for uploads,
// facilitating secure, authorized file transfers without requiring direct access to the storage credentials.
UploadTypePresigned = 2
// partSeparator is used as a delimiter in multipart upload processes,
// separating individual file parts.
partSeparator = ","
)
pkg/common/db/s3/cont/controller.go
-282
View File
@@ -1,282 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cont
import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"path"
"strings"
"time"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/google/uuid"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
)
func New(cache cache.S3Cache, impl s3.Interface) *Controller {
return &Controller{
cache: cache,
impl: impl,
}
}
type Controller struct {
cache cache.S3Cache
impl s3.Interface
}
func (c *Controller) Engine() string {
return c.impl.Engine()
}
func (c *Controller) HashPath(md5 string) string {
return path.Join(hashPath, md5)
}
func (c *Controller) NowPath() string {
now := time.Now()
return path.Join(
fmt.Sprintf("%04d", now.Year()),
fmt.Sprintf("%02d", now.Month()),
fmt.Sprintf("%02d", now.Day()),
fmt.Sprintf("%02d", now.Hour()),
fmt.Sprintf("%02d", now.Minute()),
fmt.Sprintf("%02d", now.Second()),
)
}
func (c *Controller) UUID() string {
id := uuid.New()
return hex.EncodeToString(id[:])
}
func (c *Controller) PartSize(ctx context.Context, size int64) (int64, error) {
return c.impl.PartSize(ctx, size)
}
func (c *Controller) PartLimit() *s3.PartLimit {
return c.impl.PartLimit()
}
func (c *Controller) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
return c.cache.GetKey(ctx, c.impl.Engine(), name)
}
func (c *Controller) GetHashObject(ctx context.Context, hash string) (*s3.ObjectInfo, error) {
return c.StatObject(ctx, c.HashPath(hash))
}
func (c *Controller) InitiateUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*InitiateUploadResult, error) {
defer log.ZDebug(ctx, "return")
if size < 0 {
return nil, errors.New("invalid size")
}
if hashBytes, err := hex.DecodeString(hash); err != nil {
return nil, err
} else if len(hashBytes) != md5.Size {
return nil, errors.New("invalid md5")
}
partSize, err := c.impl.PartSize(ctx, size)
if err != nil {
return nil, err
}
partNumber := int(size / partSize)
if size%partSize > 0 {
partNumber++
}
if maxParts > 0 && partNumber > 0 && partNumber < maxParts {
return nil, fmt.Errorf("too many parts: %d", partNumber)
}
if info, err := c.StatObject(ctx, c.HashPath(hash)); err == nil {
return nil, &HashAlreadyExistsError{Object: info}
} else if !c.impl.IsNotFound(err) {
return nil, err
}
if size <= partSize {
// Pre-signed upload
key := path.Join(tempPath, c.NowPath(), fmt.Sprintf("%s_%d_%s.presigned", hash, size, c.UUID()))
rawURL, err := c.impl.PresignedPutObject(ctx, key, expire)
if err != nil {
return nil, err
}
return &InitiateUploadResult{
UploadID: newMultipartUploadID(multipartUploadID{
Type: UploadTypePresigned,
ID: "",
Key: key,
Size: size,
Hash: hash,
}),
PartSize: partSize,
Sign: &s3.AuthSignResult{
Parts: []s3.SignPart{
{
PartNumber: 1,
URL: rawURL,
},
},
},
}, nil
} else {
// Fragment upload
upload, err := c.impl.InitiateMultipartUpload(ctx, c.HashPath(hash))
if err != nil {
return nil, err
}
if maxParts < 0 {
maxParts = partNumber
}
var authSign *s3.AuthSignResult
if maxParts > 0 {
partNumbers := make([]int, maxParts)
for i := 0; i < maxParts; i++ {
partNumbers[i] = i + 1
}
authSign, err = c.impl.AuthSign(ctx, upload.UploadID, upload.Key, time.Hour*24, partNumbers)
if err != nil {
return nil, err
}
}
return &InitiateUploadResult{
UploadID: newMultipartUploadID(multipartUploadID{
Type: UploadTypeMultipart,
ID: upload.UploadID,
Key: upload.Key,
Size: size,
Hash: hash,
}),
PartSize: partSize,
Sign: authSign,
}, nil
}
}
func (c *Controller) CompleteUpload(ctx context.Context, uploadID string, partHashs []string) (*UploadResult, error) {
defer log.ZDebug(ctx, "return")
upload, err := parseMultipartUploadID(uploadID)
if err != nil {
return nil, err
}
if md5Sum := md5.Sum([]byte(strings.Join(partHashs, partSeparator))); hex.EncodeToString(md5Sum[:]) != upload.Hash {
return nil, errors.New("md5 mismatching")
}
if info, err := c.StatObject(ctx, c.HashPath(upload.Hash)); err == nil {
return &UploadResult{
Key: info.Key,
Size: info.Size,
Hash: info.ETag,
}, nil
} else if !c.IsNotFound(err) {
return nil, err
}
cleanObject := make(map[string]struct{})
defer func() {
for key := range cleanObject {
_ = c.impl.DeleteObject(ctx, key)
}
}()
var targetKey string
switch upload.Type {
case UploadTypeMultipart:
parts := make([]s3.Part, len(partHashs))
for i, part := range partHashs {
parts[i] = s3.Part{
PartNumber: i + 1,
ETag: part,
}
}
// todo: Validation size
result, err := c.impl.CompleteMultipartUpload(ctx, upload.ID, upload.Key, parts)
if err != nil {
return nil, err
}
targetKey = result.Key
case UploadTypePresigned:
uploadInfo, err := c.StatObject(ctx, upload.Key)
if err != nil {
return nil, err
}
cleanObject[uploadInfo.Key] = struct{}{}
if uploadInfo.Size != upload.Size {
return nil, errors.New("upload size mismatching")
}
md5Sum := md5.Sum([]byte(strings.Join([]string{uploadInfo.ETag}, partSeparator)))
if md5val := hex.EncodeToString(md5Sum[:]); md5val != upload.Hash {
return nil, errs.ErrArgs.Wrap(fmt.Sprintf("md5 mismatching %s != %s", md5val, upload.Hash))
}
// Prevents concurrent operations at this time that cause files to be overwritten
copyInfo, err := c.impl.CopyObject(ctx, uploadInfo.Key, upload.Key+"."+c.UUID())
if err != nil {
return nil, err
}
cleanObject[copyInfo.Key] = struct{}{}
if copyInfo.ETag != uploadInfo.ETag {
return nil, errors.New("[concurrency]copy md5 mismatching")
}
hashCopyInfo, err := c.impl.CopyObject(ctx, copyInfo.Key, c.HashPath(upload.Hash))
if err != nil {
return nil, err
}
log.ZInfo(ctx, "hashCopyInfo", "value", fmt.Sprintf("%+v", hashCopyInfo))
targetKey = hashCopyInfo.Key
default:
return nil, errors.New("invalid upload id type")
}
if err := c.cache.DelS3Key(c.impl.Engine(), targetKey).ExecDel(ctx); err != nil {
return nil, err
}
return &UploadResult{
Key: targetKey,
Size: upload.Size,
Hash: upload.Hash,
}, nil
}
func (c *Controller) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) {
upload, err := parseMultipartUploadID(uploadID)
if err != nil {
return nil, err
}
switch upload.Type {
case UploadTypeMultipart:
return c.impl.AuthSign(ctx, upload.ID, upload.Key, time.Hour*24, partNumbers)
case UploadTypePresigned:
return nil, errors.New("presigned id not support auth sign")
default:
return nil, errors.New("invalid upload id type")
}
}
func (c *Controller) IsNotFound(err error) bool {
return c.impl.IsNotFound(err) || errs.ErrRecordNotFound.Is(err)
}
func (c *Controller) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
if opt.Image != nil {
opt.Filename = ""
opt.ContentType = ""
}
return c.impl.AccessURL(ctx, name, expire, opt)
}
func (c *Controller) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
return c.impl.FormData(ctx, name, size, contentType, duration)
}
pkg/common/db/s3/cont/error.go
-29
View File
@@ -1,29 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cont
import (
"fmt"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
)
type HashAlreadyExistsError struct {
Object *s3.ObjectInfo
}
func (e *HashAlreadyExistsError) Error() string {
return fmt.Sprintf("hash already exists: %s", e.Object.Key)
}
pkg/common/db/s3/cont/id.go
-49
View File
@@ -1,49 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cont
import (
"encoding/base64"
"encoding/json"
"fmt"
)
type multipartUploadID struct {
Type int `json:"a,omitempty"`
ID string `json:"b,omitempty"`
Key string `json:"c,omitempty"`
Size int64 `json:"d,omitempty"`
Hash string `json:"e,omitempty"`
}
func newMultipartUploadID(id multipartUploadID) string {
data, err := json.Marshal(id)
if err != nil {
panic(err)
}
return base64.StdEncoding.EncodeToString(data)
}
func parseMultipartUploadID(id string) (*multipartUploadID, error) {
data, err := base64.StdEncoding.DecodeString(id)
if err != nil {
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
}
var upload multipartUploadID
if err := json.Unmarshal(data, &upload); err != nil {
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
}
return &upload, nil
}
pkg/common/db/s3/cont/structs.go
-34
View File
@@ -1,34 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cont
import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
type InitiateUploadResult struct {
// UploadID uniquely identifies the upload session for tracking and management purposes.
UploadID string `json:"uploadID"`
// PartSize specifies the size of each part in a multipart upload. This is relevant for breaking down large uploads into manageable pieces.
PartSize int64 `json:"partSize"`
// Sign contains the authentication and signature information necessary for securely uploading each part. This could include signed URLs or tokens.
Sign *s3.AuthSignResult `json:"sign"`
}
type UploadResult struct {
Hash string `json:"hash"`
Size int64 `json:"size"`
Key string `json:"key"`
}
pkg/common/db/s3/cos/cos.go
-400
View File
@@ -1,400 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cos
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/OpenIMSDK/tools/errs"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
"github.com/tencentyun/cos-go-sdk-v5"
)
const (
minPartSize int64 = 1024 * 1024 * 1 // 1MB
maxPartSize int64 = 1024 * 1024 * 1024 * 5 // 5GB
maxNumSize int64 = 1000
)
const (
imagePng = "png"
imageJpg = "jpg"
imageJpeg = "jpeg"
imageGif = "gif"
imageWebp = "webp"
)
const successCode = http.StatusOK
type Config struct {
BucketURL string
SecretID string
SecretKey string
SessionToken string
PublicRead bool
}
func NewCos(conf Config) (s3.Interface, error) {
u, err := url.Parse(conf.BucketURL)
if err != nil {
panic(err)
}
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: conf.SecretID,
SecretKey: conf.SecretKey,
SessionToken: conf.SessionToken,
},
})
return &Cos{
publicRead: conf.PublicRead,
copyURL: u.Host + "/",
client: client,
credential: client.GetCredential(),
}, nil
}
type Cos struct {
publicRead bool
copyURL string
client *cos.Client
credential *cos.Credential
}
func (c *Cos) Engine() string {
return "tencent-cos"
}
func (c *Cos) PartLimit() *s3.PartLimit {
return &s3.PartLimit{
MinPartSize: minPartSize,
MaxPartSize: maxPartSize,
MaxNumSize: maxNumSize,
}
}
func (c *Cos) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
result, _, err := c.client.Object.InitiateMultipartUpload(ctx, name, nil)
if err != nil {
return nil, err
}
return &s3.InitiateMultipartUploadResult{
UploadID: result.UploadID,
Bucket: result.Bucket,
Key: result.Key,
}, nil
}
func (c *Cos) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
opts := &cos.CompleteMultipartUploadOptions{
Parts: make([]cos.Object, len(parts)),
}
for i, part := range parts {
opts.Parts[i] = cos.Object{
PartNumber: part.PartNumber,
ETag: strings.ReplaceAll(part.ETag, `"`, ``),
}
}
result, _, err := c.client.Object.CompleteMultipartUpload(ctx, name, uploadID, opts)
if err != nil {
return nil, err
}
return &s3.CompleteMultipartUploadResult{
Location: result.Location,
Bucket: result.Bucket,
Key: result.Key,
ETag: result.ETag,
}, nil
}
func (c *Cos) PartSize(ctx context.Context, size int64) (int64, error) {
if size <= 0 {
return 0, errors.New("size must be greater than 0")
}
if size > maxPartSize*maxNumSize {
return 0, fmt.Errorf("COS size must be less than the maximum allowed limit")
}
if size <= minPartSize*maxNumSize {
return minPartSize, nil
}
partSize := size / maxNumSize
if size%maxNumSize != 0 {
partSize++
}
return partSize, nil
}
func (c *Cos) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
result := s3.AuthSignResult{
URL: c.client.BaseURL.BucketURL.String() + "/" + cos.EncodeURIComponent(name),
Query: url.Values{"uploadId": {uploadID}},
Header: make(http.Header),
Parts: make([]s3.SignPart, len(partNumbers)),
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, result.URL, nil)
if err != nil {
return nil, err
}
cos.AddAuthorizationHeader(c.credential.SecretID, c.credential.SecretKey, c.credential.SessionToken, req, cos.NewAuthTime(expire))
result.Header = req.Header
for i, partNumber := range partNumbers {
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
}
}
return &result, nil
}
func (c *Cos) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodPut, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
if err != nil {
return "", err
}
return rawURL.String(), nil
}
func (c *Cos) DeleteObject(ctx context.Context, name string) error {
_, err := c.client.Object.Delete(ctx, name)
return err
}
func (c *Cos) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
if name != "" && name[0] == '/' {
name = name[1:]
}
info, err := c.client.Object.Head(ctx, name, nil)
if err != nil {
return nil, err
}
res := &s3.ObjectInfo{Key: name}
if res.ETag = strings.ToLower(strings.ReplaceAll(info.Header.Get("ETag"), `"`, "")); res.ETag == "" {
return nil, errors.New("StatObject etag not found")
}
if contentLengthStr := info.Header.Get("Content-Length"); contentLengthStr == "" {
return nil, errors.New("StatObject content-length not found")
} else {
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return nil, fmt.Errorf("StatObject content-length parse error: %w", err)
}
if res.Size < 0 {
return nil, errors.New("StatObject content-length must be greater than 0")
}
}
if lastModified := info.Header.Get("Last-Modified"); lastModified == "" {
return nil, errors.New("StatObject last-modified not found")
} else {
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
if err != nil {
return nil, fmt.Errorf("StatObject last-modified parse error: %w", err)
}
}
return res, nil
}
func (c *Cos) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
sourceURL := c.copyURL + src
result, _, err := c.client.Object.Copy(ctx, dst, sourceURL, nil)
if err != nil {
return nil, err
}
return &s3.CopyObjectInfo{
Key: dst,
ETag: strings.ReplaceAll(result.ETag, `"`, ``),
}, nil
}
func (c *Cos) IsNotFound(err error) bool {
switch e := errs.Unwrap(err).(type) {
case *cos.ErrorResponse:
return e.Response.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (c *Cos) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
_, err := c.client.Object.AbortMultipartUpload(ctx, name, uploadID)
return err
}
func (c *Cos) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
result, _, err := c.client.Object.ListParts(ctx, name, uploadID, &cos.ObjectListPartsOptions{
MaxParts: strconv.Itoa(maxParts),
PartNumberMarker: strconv.Itoa(partNumberMarker),
})
if err != nil {
return nil, err
}
res := &s3.ListUploadedPartsResult{
Key: result.Key,
UploadID: result.UploadID,
UploadedParts: make([]s3.UploadedPart, len(result.Parts)),
}
res.MaxParts, _ = strconv.Atoi(result.MaxParts)
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
for i, part := range result.Parts {
lastModified, _ := time.Parse(http.TimeFormat, part.LastModified)
res.UploadedParts[i] = s3.UploadedPart{
PartNumber: part.PartNumber,
LastModified: lastModified,
ETag: part.ETag,
Size: part.Size,
}
}
return res, nil
}
func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
var imageMogr string
var option cos.PresignedURLOptions
if opt != nil {
query := make(url.Values)
if opt.Image != nil {
// https://cloud.tencent.com/document/product/436/44880
style := make([]string, 0, 2)
wh := make([]string, 2)
if opt.Image.Width > 0 {
wh[0] = strconv.Itoa(opt.Image.Width)
}
if opt.Image.Height > 0 {
wh[1] = strconv.Itoa(opt.Image.Height)
}
if opt.Image.Width > 0 || opt.Image.Height > 0 {
style = append(style, strings.Join(wh, "x"))
}
switch opt.Image.Format {
case
imagePng,
imageJpg,
imageJpeg,
imageGif,
imageWebp:
style = append(style, "format/"+opt.Image.Format)
}
if len(style) > 0 {
imageMogr = "imageMogr2/thumbnail/" + strings.Join(style, "/") + "/ignore-error/1"
}
}
if opt.ContentType != "" {
query.Set("response-content-type", opt.ContentType)
}
if opt.Filename != "" {
query.Set("response-content-disposition", `attachment; filename=`+strconv.Quote(opt.Filename))
}
if len(query) > 0 {
option.Query = &query
}
}
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
expire = time.Second
}
rawURL, err := c.getPresignedURL(ctx, name, expire, &option)
if err != nil {
return "", err
}
if imageMogr != "" {
if rawURL.RawQuery == "" {
rawURL.RawQuery = imageMogr
} else {
rawURL.RawQuery = rawURL.RawQuery + "&" + imageMogr
}
}
return rawURL.String(), nil
}
func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Duration, opt *cos.PresignedURLOptions) (*url.URL, error) {
if !c.publicRead {
return c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, opt)
}
return c.client.Object.GetObjectURL(name), nil
}
func (c *Cos) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
// https://cloud.tencent.com/document/product/436/14690
now := time.Now()
expiration := now.Add(duration)
keyTime := fmt.Sprintf("%d;%d", now.Unix(), expiration.Unix())
conditions := []any{
map[string]string{"q-sign-algorithm": "sha1"},
map[string]string{"q-ak": c.credential.SecretID},
map[string]string{"q-sign-time": keyTime},
map[string]string{"key": name},
}
if contentType != "" {
conditions = append(conditions, map[string]string{"Content-Type": contentType})
}
policy := map[string]any{
"expiration": expiration.Format("2006-01-02T15:04:05.000Z"),
"conditions": conditions,
}
policyJson, err := json.Marshal(policy)
if err != nil {
return nil, err
}
signKey := hmacSha1val(c.credential.SecretKey, keyTime)
strToSign := sha1val(string(policyJson))
signature := hmacSha1val(signKey, strToSign)
fd := &s3.FormData{
URL: c.client.BaseURL.BucketURL.String(),
File: "file",
Expires: expiration,
FormData: map[string]string{
"policy": base64.StdEncoding.EncodeToString(policyJson),
"q-sign-algorithm": "sha1",
"q-ak": c.credential.SecretID,
"q-key-time": keyTime,
"q-signature": signature,
"key": name,
"success_action_status": strconv.Itoa(successCode),
},
SuccessCodes: []int{successCode},
}
if contentType != "" {
fd.FormData["Content-Type"] = contentType
}
if c.credential.SessionToken != "" {
fd.FormData["x-cos-security-token"] = c.credential.SessionToken
}
return fd, nil
}
func hmacSha1val(key, msg string) string {
v := hmac.New(sha1.New, []byte(key))
v.Write([]byte(msg))
return hex.EncodeToString(v.Sum(nil))
}
func sha1val(msg string) string {
sha1Hash := sha1.New()
sha1Hash.Write([]byte(msg))
return hex.EncodeToString(sha1Hash.Sum(nil))
}
pkg/common/db/s3/cos/internal.go
-27
View File
@@ -1,27 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cos
import (
"context"
"net/http"
"net/url"
_ "unsafe"
"github.com/tencentyun/cos-go-sdk-v5"
)
//go:linkname newRequest github.com/tencentyun/cos-go-sdk-v5.(*Client).newRequest
func newRequest(c *cos.Client, ctx context.Context, baseURL *url.URL, uri, method string, body any, optQuery any, optHeader any) (req *http.Request, err error)
pkg/common/db/s3/minio/image.go
-120
View File
@@ -1,120 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package minio
import (
"image"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"io"
_ "golang.org/x/image/bmp"
_ "golang.org/x/image/tiff"
_ "golang.org/x/image/webp"
)
const (
formatPng = "png"
formatJpeg = "jpeg"
formatJpg = "jpg"
formatGif = "gif"
)
func ImageStat(reader io.Reader) (image.Image, string, error) {
return image.Decode(reader)
}
func ImageWidthHeight(img image.Image) (int, int) {
bounds := img.Bounds().Max
return bounds.X, bounds.Y
}
func resizeImage(img image.Image, maxWidth, maxHeight int) image.Image {
bounds := img.Bounds()
imgWidth := bounds.Max.X
imgHeight := bounds.Max.Y
// Calculating scaling
scaleWidth := float64(maxWidth) / float64(imgWidth)
scaleHeight := float64(maxHeight) / float64(imgHeight)
// If both are 0, then no scaling is done and the original image is returned
if maxWidth == 0 && maxHeight == 0 {
return img
}
// If both width and height are greater than 0, select a smaller zoom ratio to maintain the aspect ratio
if maxWidth > 0 && maxHeight > 0 {
scale := scaleWidth
if scaleHeight < scaleWidth {
scale = scaleHeight
}
// Calculate Thumbnail Size
thumbnailWidth := int(float64(imgWidth) * scale)
thumbnailHeight := int(float64(imgHeight) * scale)
// Thumbnails are generated using the Resample method of the "image" library.
thumbnail := image.NewRGBA(image.Rect(0, 0, thumbnailWidth, thumbnailHeight))
for y := 0; y < thumbnailHeight; y++ {
for x := 0; x < thumbnailWidth; x++ {
srcX := int(float64(x) / scale)
srcY := int(float64(y) / scale)
thumbnail.Set(x, y, img.At(srcX, srcY))
}
}
return thumbnail
}
// If only width or height is specified, thumbnails are generated based on the maximum not to exceed rule
if maxWidth > 0 {
thumbnailWidth := maxWidth
thumbnailHeight := int(float64(imgHeight) * scaleWidth)
// Thumbnails are generated using the Resample method of the "image" library.
thumbnail := image.NewRGBA(image.Rect(0, 0, thumbnailWidth, thumbnailHeight))
for y := 0; y < thumbnailHeight; y++ {
for x := 0; x < thumbnailWidth; x++ {
srcX := int(float64(x) / scaleWidth)
srcY := int(float64(y) / scaleWidth)
thumbnail.Set(x, y, img.At(srcX, srcY))
}
}
return thumbnail
}
if maxHeight > 0 {
thumbnailWidth := int(float64(imgWidth) * scaleHeight)
thumbnailHeight := maxHeight
// Thumbnails are generated using the Resample method of the "image" library.
thumbnail := image.NewRGBA(image.Rect(0, 0, thumbnailWidth, thumbnailHeight))
for y := 0; y < thumbnailHeight; y++ {
for x := 0; x < thumbnailWidth; x++ {
srcX := int(float64(x) / scaleHeight)
srcY := int(float64(y) / scaleHeight)
thumbnail.Set(x, y, img.At(srcX, srcY))
}
}
return thumbnail
}
// By default, the original image is returned
return img
}
pkg/common/db/s3/minio/internal.go
-25
View File
@@ -1,25 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package minio
import (
"net/url"
_ "unsafe"
"github.com/minio/minio-go/v7"
)
//go:linkname makeTargetURL github.com/minio/minio-go/v7.(*Client).makeTargetURL
func makeTargetURL(client *minio.Client, bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error)
pkg/common/db/s3/minio/minio.go
-499
View File
@@ -1,499 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package minio
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"reflect"
"strconv"
"strings"
"sync"
"time"
"unsafe"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/signer"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
)
const (
unsignedPayload = "UNSIGNED-PAYLOAD"
)
const (
minPartSize int64 = 1024 * 1024 * 5 // 5MB
maxPartSize int64 = 1024 * 1024 * 1024 * 5 // 5GB
maxNumSize int64 = 10000
)
const (
maxImageWidth = 1024
maxImageHeight = 1024
maxImageSize = 1024 * 1024 * 50
imageThumbnailPath = "openim/thumbnail"
)
const successCode = http.StatusOK
type Config struct {
Bucket string
Endpoint string
AccessKeyID string
SecretAccessKey string
SessionToken string
SignEndpoint string
PublicRead bool
}
func NewMinio(cache cache.MinioCache, conf Config) (s3.Interface, error) {
u, err := url.Parse(conf.Endpoint)
if err != nil {
return nil, err
}
opts := &minio.Options{
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken),
Secure: u.Scheme == "https",
}
client, err := minio.New(u.Host, opts)
if err != nil {
return nil, err
}
m := &Minio{
conf: conf,
bucket: conf.Bucket,
core: &minio.Core{Client: client},
lock: &sync.Mutex{},
init: false,
cache: cache,
}
if conf.SignEndpoint == "" || conf.SignEndpoint == conf.Endpoint {
m.opts = opts
m.sign = m.core.Client
m.prefix = u.Path
u.Path = ""
conf.Endpoint = u.String()
m.signEndpoint = conf.Endpoint
} else {
su, err := url.Parse(conf.SignEndpoint)
if err != nil {
return nil, err
}
m.opts = &minio.Options{
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken),
Secure: su.Scheme == "https",
}
m.sign, err = minio.New(su.Host, m.opts)
if err != nil {
return nil, err
}
m.prefix = su.Path
su.Path = ""
conf.SignEndpoint = su.String()
m.signEndpoint = conf.SignEndpoint
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := m.initMinio(ctx); err != nil {
fmt.Println("init minio error:", err)
}
return m, nil
}
type Minio struct {
conf Config
bucket string
signEndpoint string
location string
opts *minio.Options
core *minio.Core
sign *minio.Client
lock sync.Locker
init bool
prefix string
cache cache.MinioCache
}
func (m *Minio) initMinio(ctx context.Context) error {
if m.init {
return nil
}
m.lock.Lock()
defer m.lock.Unlock()
if m.init {
return nil
}
exists, err := m.core.Client.BucketExists(ctx, m.conf.Bucket)
if err != nil {
return fmt.Errorf("check bucket exists error: %w", err)
}
if !exists {
if err = m.core.Client.MakeBucket(ctx, m.conf.Bucket, minio.MakeBucketOptions{}); err != nil {
return fmt.Errorf("make bucket error: %w", err)
}
}
if m.conf.PublicRead {
policy := fmt.Sprintf(
`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`,
m.conf.Bucket,
)
if err = m.core.Client.SetBucketPolicy(ctx, m.conf.Bucket, policy); err != nil {
return err
}
}
m.location, err = m.core.Client.GetBucketLocation(ctx, m.conf.Bucket)
if err != nil {
return err
}
func() {
if m.conf.SignEndpoint == "" || m.conf.SignEndpoint == m.conf.Endpoint {
return
}
defer func() {
if r := recover(); r != nil {
m.sign = m.core.Client
log.ZWarn(
context.Background(),
"set sign bucket location cache panic",
errors.New("failed to get private field value"),
"recover",
fmt.Sprintf("%+v", r),
"development version",
"github.com/minio/minio-go/v7 v7.0.61",
)
}
}()
blc := reflect.ValueOf(m.sign).Elem().FieldByName("bucketLocCache")
vblc := reflect.New(reflect.PtrTo(blc.Type()))
*(*unsafe.Pointer)(vblc.UnsafePointer()) = unsafe.Pointer(blc.UnsafeAddr())
vblc.Elem().Elem().Interface().(interface{ Set(string, string) }).Set(m.conf.Bucket, m.location)
}()
m.init = true
return nil
}
func (m *Minio) Engine() string {
return "minio"
}
func (m *Minio) PartLimit() *s3.PartLimit {
return &s3.PartLimit{
MinPartSize: minPartSize,
MaxPartSize: maxPartSize,
MaxNumSize: maxNumSize,
}
}
func (m *Minio) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
if err := m.initMinio(ctx); err != nil {
return nil, err
}
uploadID, err := m.core.NewMultipartUpload(ctx, m.bucket, name, minio.PutObjectOptions{})
if err != nil {
return nil, err
}
return &s3.InitiateMultipartUploadResult{
Bucket: m.bucket,
Key: name,
UploadID: uploadID,
}, nil
}
func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
if err := m.initMinio(ctx); err != nil {
return nil, err
}
minioParts := make([]minio.CompletePart, len(parts))
for i, part := range parts {
minioParts[i] = minio.CompletePart{
PartNumber: part.PartNumber,
ETag: strings.ToLower(part.ETag),
}
}
upload, err := m.core.CompleteMultipartUpload(ctx, m.bucket, name, uploadID, minioParts, minio.PutObjectOptions{})
if err != nil {
return nil, err
}
m.delObjectImageInfoKey(ctx, name, upload.Size)
return &s3.CompleteMultipartUploadResult{
Location: upload.Location,
Bucket: upload.Bucket,
Key: upload.Key,
ETag: strings.ToLower(upload.ETag),
}, nil
}
func (m *Minio) PartSize(ctx context.Context, size int64) (int64, error) {
if size <= 0 {
return 0, errors.New("size must be greater than 0")
}
if size > maxPartSize*maxNumSize {
return 0, fmt.Errorf("MINIO size must be less than the maximum allowed limit")
}
if size <= minPartSize*maxNumSize {
return minPartSize, nil
}
partSize := size / maxNumSize
if size%maxNumSize != 0 {
partSize++
}
return partSize, nil
}
func (m *Minio) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
if err := m.initMinio(ctx); err != nil {
return nil, err
}
creds, err := m.opts.Creds.Get()
if err != nil {
return nil, err
}
result := s3.AuthSignResult{
URL: m.signEndpoint + "/" + m.bucket + "/" + name,
Query: url.Values{"uploadId": {uploadID}},
Parts: make([]s3.SignPart, len(partNumbers)),
}
for i, partNumber := range partNumbers {
rawURL := result.URL + "?partNumber=" + strconv.Itoa(partNumber) + "&uploadId=" + uploadID
request, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, nil)
if err != nil {
return nil, err
}
request.Header.Set("X-Amz-Content-Sha256", unsignedPayload)
request = signer.SignV4Trailer(*request, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken, m.location, nil)
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
Header: request.Header,
}
}
if m.prefix != "" {
result.URL = m.signEndpoint + m.prefix + "/" + m.bucket + "/" + name
}
return &result, nil
}
func (m *Minio) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
if err := m.initMinio(ctx); err != nil {
return "", err
}
rawURL, err := m.sign.PresignedPutObject(ctx, m.bucket, name, expire)
if err != nil {
return "", err
}
if m.prefix != "" {
rawURL.Path = path.Join(m.prefix, rawURL.Path)
}
return rawURL.String(), nil
}
func (m *Minio) DeleteObject(ctx context.Context, name string) error {
if err := m.initMinio(ctx); err != nil {
return err
}
return m.core.Client.RemoveObject(ctx, m.bucket, name, minio.RemoveObjectOptions{})
}
func (m *Minio) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
if err := m.initMinio(ctx); err != nil {
return nil, err
}
info, err := m.core.Client.StatObject(ctx, m.bucket, name, minio.StatObjectOptions{})
if err != nil {
return nil, err
}
return &s3.ObjectInfo{
ETag: strings.ToLower(info.ETag),
Key: info.Key,
Size: info.Size,
LastModified: info.LastModified,
}, nil
}
func (m *Minio) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
if err := m.initMinio(ctx); err != nil {
return nil, err
}
result, err := m.core.Client.CopyObject(ctx, minio.CopyDestOptions{
Bucket: m.bucket,
Object: dst,
}, minio.CopySrcOptions{
Bucket: m.bucket,
Object: src,
})
if err != nil {
return nil, err
}
return &s3.CopyObjectInfo{
Key: dst,
ETag: strings.ToLower(result.ETag),
}, nil
}
func (m *Minio) IsNotFound(err error) bool {
switch e := errs.Unwrap(err).(type) {
case minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
case *minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (m *Minio) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
if err := m.initMinio(ctx); err != nil {
return err
}
return m.core.AbortMultipartUpload(ctx, m.bucket, name, uploadID)
}
func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
if err := m.initMinio(ctx); err != nil {
return nil, err
}
result, err := m.core.ListObjectParts(ctx, m.bucket, name, uploadID, partNumberMarker, maxParts)
if err != nil {
return nil, err
}
res := &s3.ListUploadedPartsResult{
Key: result.Key,
UploadID: result.UploadID,
MaxParts: result.MaxParts,
NextPartNumberMarker: result.NextPartNumberMarker,
UploadedParts: make([]s3.UploadedPart, len(result.ObjectParts)),
}
for i, part := range result.ObjectParts {
res.UploadedParts[i] = s3.UploadedPart{
PartNumber: part.PartNumber,
LastModified: part.LastModified,
ETag: part.ETag,
Size: part.Size,
}
}
return res, nil
}
func (m *Minio) PresignedGetObject(ctx context.Context, name string, expire time.Duration, query url.Values) (string, error) {
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
expire = time.Second
}
var (
rawURL *url.URL
err error
)
if m.conf.PublicRead {
rawURL, err = makeTargetURL(m.sign, m.bucket, name, m.location, false, query)
} else {
rawURL, err = m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query)
}
if err != nil {
return "", err
}
if m.prefix != "" {
rawURL.Path = path.Join(m.prefix, rawURL.Path)
}
return rawURL.String(), nil
}
func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
if err := m.initMinio(ctx); err != nil {
return "", err
}
reqParams := make(url.Values)
if opt != nil {
if opt.ContentType != "" {
reqParams.Set("response-content-type", opt.ContentType)
}
if opt.Filename != "" {
reqParams.Set("response-content-disposition", `attachment; filename=`+strconv.Quote(opt.Filename))
}
}
if opt.Image == nil || (opt.Image.Width < 0 && opt.Image.Height < 0 && opt.Image.Format == "") || (opt.Image.Width > maxImageWidth || opt.Image.Height > maxImageHeight) {
return m.PresignedGetObject(ctx, name, expire, reqParams)
}
return m.getImageThumbnailURL(ctx, name, expire, opt.Image)
}
func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]byte, error) {
object, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
defer object.Close()
if limit < 0 {
return io.ReadAll(object)
}
return io.ReadAll(io.LimitReader(object, limit))
}
func (m *Minio) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
if err := m.initMinio(ctx); err != nil {
return nil, err
}
policy := minio.NewPostPolicy()
if err := policy.SetKey(name); err != nil {
return nil, err
}
expires := time.Now().Add(duration)
if err := policy.SetExpires(expires); err != nil {
return nil, err
}
if size > 0 {
if err := policy.SetContentLengthRange(0, size); err != nil {
return nil, err
}
}
if err := policy.SetSuccessStatusAction(strconv.Itoa(successCode)); err != nil {
return nil, err
}
if contentType != "" {
if err := policy.SetContentType(contentType); err != nil {
return nil, err
}
}
if err := policy.SetBucket(m.bucket); err != nil {
return nil, err
}
u, fd, err := m.core.PresignedPostPolicy(ctx, policy)
if err != nil {
return nil, err
}
sign, err := url.Parse(m.signEndpoint)
if err != nil {
return nil, err
}
u.Scheme = sign.Scheme
u.Host = sign.Host
return &s3.FormData{
URL: u.String(),
File: "file",
Header: nil,
FormData: fd,
Expires: expires,
SuccessCodes: []int{successCode},
}, nil
}
pkg/common/db/s3/minio/thumbnail.go
-150
View File
@@ -1,150 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package minio
import (
"bytes"
"context"
"errors"
"fmt"
"image"
"image/gif"
"image/jpeg"
"image/png"
"net/url"
"path/filepath"
"strings"
"time"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/minio/minio-go/v7"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
)
func (m *Minio) getImageThumbnailURL(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
var img image.Image
info, err := m.cache.GetImageObjectKeyInfo(ctx, name, func(ctx context.Context) (info *cache.MinioImageInfo, err error) {
info, img, err = m.getObjectImageInfo(ctx, name)
return
})
if err != nil {
return "", err
}
if !info.IsImg {
return "", errs.ErrData.Wrap("object not image")
}
if opt.Width > info.Width || opt.Width <= 0 {
opt.Width = info.Width
}
if opt.Height > info.Height || opt.Height <= 0 {
opt.Height = info.Height
}
opt.Format = strings.ToLower(opt.Format)
if opt.Format == formatJpg {
opt.Format = formatJpeg
}
switch opt.Format {
case formatPng, formatJpeg, formatGif:
default:
opt.Format = ""
}
reqParams := make(url.Values)
if opt.Width == info.Width && opt.Height == info.Height && (opt.Format == info.Format || opt.Format == "") {
reqParams.Set("response-content-type", "image/"+info.Format)
return m.PresignedGetObject(ctx, name, expire, reqParams)
}
if opt.Format == "" {
switch opt.Format {
case formatGif:
opt.Format = formatGif
case formatJpeg:
opt.Format = formatJpeg
case formatPng:
opt.Format = formatPng
default:
opt.Format = formatPng
}
}
key, err := m.cache.GetThumbnailKey(ctx, name, opt.Format, opt.Width, opt.Height, func(ctx context.Context) (string, error) {
if img == nil {
var reader *minio.Object
reader, err = m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
if err != nil {
return "", err
}
defer reader.Close()
img, _, err = ImageStat(reader)
if err != nil {
return "", err
}
}
thumbnail := resizeImage(img, opt.Width, opt.Height)
buf := bytes.NewBuffer(nil)
switch opt.Format {
case formatPng:
err = png.Encode(buf, thumbnail)
case formatJpeg:
err = jpeg.Encode(buf, thumbnail, nil)
case formatGif:
err = gif.Encode(buf, thumbnail, nil)
}
cacheKey := filepath.Join(imageThumbnailPath, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
if _, err = m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
return "", err
}
return cacheKey, nil
})
if err != nil {
return "", err
}
reqParams.Set("response-content-type", "image/"+opt.Format)
return m.PresignedGetObject(ctx, key, expire, reqParams)
}
func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.MinioImageInfo, image.Image, error) {
fileInfo, err := m.StatObject(ctx, name)
if err != nil {
return nil, nil, err
}
if fileInfo.Size > maxImageSize {
return nil, nil, errors.New("file size too large")
}
imageData, err := m.getObjectData(ctx, name, fileInfo.Size)
if err != nil {
return nil, nil, err
}
var info cache.MinioImageInfo
imageInfo, format, err := ImageStat(bytes.NewReader(imageData))
if err == nil {
info.IsImg = true
info.Format = format
info.Width, info.Height = ImageWidthHeight(imageInfo)
} else {
info.IsImg = false
}
info.Etag = fileInfo.ETag
return &info, imageInfo, nil
}
func (m *Minio) delObjectImageInfoKey(ctx context.Context, key string, size int64) {
if size > 0 && size > maxImageSize {
return
}
if err := m.cache.DelObjectImageInfoKey(key).ExecDel(ctx); err != nil {
log.ZError(ctx, "DelObjectImageInfoKey failed", err, "key", key)
}
}
pkg/common/db/s3/oss/internal.go
-39
View File
@@ -1,39 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package oss
import (
"net/http"
"net/url"
_ "unsafe"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
)
//go:linkname signHeader github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.signHeader
func signHeader(c oss.Conn, req *http.Request, canonicalizedResource string)
//go:linkname getURLParams github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.getURLParams
func getURLParams(c oss.Conn, params map[string]any) string
//go:linkname getURL github.com/aliyun/aliyun-oss-go-sdk/oss.urlMaker.getURL
func getURL(um urlMaker, bucket, object, params string) *url.URL
type urlMaker struct {
Scheme string
NetLoc string
Type int
IsProxy bool
}
pkg/common/db/s3/oss/oss.go
-382
View File
@@ -1,382 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package oss
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"reflect"
"strconv"
"strings"
"time"
"github.com/OpenIMSDK/tools/errs"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
)
const (
minPartSize int64 = 1024 * 1024 * 1 // 1MB
maxPartSize int64 = 1024 * 1024 * 1024 * 5 // 5GB
maxNumSize int64 = 10000
)
const (
imagePng = "png"
imageJpg = "jpg"
imageJpeg = "jpeg"
imageGif = "gif"
imageWebp = "webp"
)
const successCode = http.StatusOK
type Config struct {
Endpoint string
Bucket string
BucketURL string
AccessKeyID string
AccessKeySecret string
SessionToken string
PublicRead bool
}
func NewOSS(conf Config) (s3.Interface, error) {
if conf.BucketURL == "" {
return nil, errs.Wrap(errors.New("bucket url is empty"))
}
client, err := oss.New(conf.Endpoint, conf.AccessKeyID, conf.AccessKeySecret)
if err != nil {
return nil, err
}
bucket, err := client.Bucket(conf.Bucket)
if err != nil {
return nil, errs.Wrap(err, "ali-oss bucket error")
}
if conf.BucketURL[len(conf.BucketURL)-1] != '/' {
conf.BucketURL += "/"
}
return &OSS{
bucketURL: conf.BucketURL,
bucket: bucket,
credentials: client.Config.GetCredentials(),
um: *(*urlMaker)(reflect.ValueOf(bucket.Client.Conn).Elem().FieldByName("url").UnsafePointer()),
publicRead: conf.PublicRead,
}, nil
}
type OSS struct {
bucketURL string
bucket *oss.Bucket
credentials oss.Credentials
um urlMaker
publicRead bool
}
func (o *OSS) Engine() string {
return "ali-oss"
}
func (o *OSS) PartLimit() *s3.PartLimit {
return &s3.PartLimit{
MinPartSize: minPartSize,
MaxPartSize: maxPartSize,
MaxNumSize: maxNumSize,
}
}
func (o *OSS) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
result, err := o.bucket.InitiateMultipartUpload(name)
if err != nil {
return nil, err
}
return &s3.InitiateMultipartUploadResult{
UploadID: result.UploadID,
Bucket: result.Bucket,
Key: result.Key,
}, nil
}
func (o *OSS) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
ossParts := make([]oss.UploadPart, len(parts))
for i, part := range parts {
ossParts[i] = oss.UploadPart{
PartNumber: part.PartNumber,
ETag: strings.ToUpper(part.ETag),
}
}
result, err := o.bucket.CompleteMultipartUpload(oss.InitiateMultipartUploadResult{
UploadID: uploadID,
Bucket: o.bucket.BucketName,
Key: name,
}, ossParts)
if err != nil {
return nil, err
}
return &s3.CompleteMultipartUploadResult{
Location: result.Location,
Bucket: result.Bucket,
Key: result.Key,
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
}, nil
}
func (o *OSS) PartSize(ctx context.Context, size int64) (int64, error) {
if size <= 0 {
return 0, errs.Wrap(errors.New("size must be greater than 0"))
}
if size > maxPartSize*maxNumSize {
return 0, errs.Wrap(errors.New("size must be less than the maximum allowed limit"))
}
if size <= minPartSize*maxNumSize {
return minPartSize, nil
}
partSize := size / maxNumSize
if size%maxNumSize != 0 {
partSize++
}
return partSize, nil
}
func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
result := s3.AuthSignResult{
URL: o.bucketURL + name,
Query: url.Values{"uploadId": {uploadID}},
Header: make(http.Header),
Parts: make([]s3.SignPart, len(partNumbers)),
}
for i, partNumber := range partNumbers {
rawURL := fmt.Sprintf(`%s%s?partNumber=%d&uploadId=%s`, o.bucketURL, name, partNumber, uploadID)
request, err := http.NewRequest(http.MethodPut, rawURL, nil)
if err != nil {
return nil, err
}
if o.credentials.GetSecurityToken() != "" {
request.Header.Set(oss.HTTPHeaderOssSecurityToken, o.credentials.GetSecurityToken())
}
now := time.Now().UTC().Format(http.TimeFormat)
request.Header.Set(oss.HTTPHeaderHost, request.Host)
request.Header.Set(oss.HTTPHeaderDate, now)
request.Header.Set(oss.HttpHeaderOssDate, now)
signHeader(*o.bucket.Client.Conn, request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID))
delete(request.Header, oss.HTTPHeaderDate)
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
URL: request.URL.String(),
Header: request.Header,
}
}
return &result, nil
}
func (o *OSS) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
return o.bucket.SignURL(name, http.MethodPut, int64(expire/time.Second))
}
func (o *OSS) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
header, err := o.bucket.GetObjectMeta(name)
if err != nil {
return nil, err
}
res := &s3.ObjectInfo{Key: name}
if res.ETag = strings.ToLower(strings.ReplaceAll(header.Get("ETag"), `"`, ``)); res.ETag == "" {
return nil, errs.Wrap(errors.New("StatObject etag not found"))
}
if contentLengthStr := header.Get("Content-Length"); contentLengthStr == "" {
return nil, errors.New("StatObject content-length not found")
} else {
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return nil, errs.Wrap(err, "StatObject content-length parse error")
}
if res.Size < 0 {
return nil, errs.Wrap(errors.New("StatObject content-length must be greater than 0"))
}
}
if lastModified := header.Get("Last-Modified"); lastModified == "" {
return nil, errs.Wrap(errors.New("StatObject last-modified not found"))
} else {
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
if err != nil {
return nil, errs.Wrap(err, "StatObject last-modified parse error")
}
}
return res, nil
}
func (o *OSS) DeleteObject(ctx context.Context, name string) error {
return o.bucket.DeleteObject(name)
}
func (o *OSS) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
result, err := o.bucket.CopyObject(src, dst)
if err != nil {
return nil, errs.Wrap(err, "CopyObject error")
}
return &s3.CopyObjectInfo{
Key: dst,
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
}, nil
}
func (o *OSS) IsNotFound(err error) bool {
switch e := errs.Unwrap(err).(type) {
case oss.ServiceError:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
case *oss.ServiceError:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (o *OSS) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
return o.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{
UploadID: uploadID,
Key: name,
Bucket: o.bucket.BucketName,
})
}
func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
result, err := o.bucket.ListUploadedParts(oss.InitiateMultipartUploadResult{
UploadID: uploadID,
Key: name,
Bucket: o.bucket.BucketName,
}, oss.MaxUploads(100), oss.MaxParts(maxParts), oss.PartNumberMarker(partNumberMarker))
if err != nil {
return nil, errs.Wrap(err, "ListUploadedParts error")
}
res := &s3.ListUploadedPartsResult{
Key: result.Key,
UploadID: result.UploadID,
MaxParts: result.MaxParts,
UploadedParts: make([]s3.UploadedPart, len(result.UploadedParts)),
}
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
for i, part := range result.UploadedParts {
res.UploadedParts[i] = s3.UploadedPart{
PartNumber: part.PartNumber,
LastModified: part.LastModified,
ETag: part.ETag,
Size: int64(part.Size),
}
}
return res, nil
}
func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
var opts []oss.Option
if opt != nil {
if opt.Image != nil {
// Docs Address: https://help.aliyun.com/zh/oss/user-guide/resize-images-4?spm=a2c4g.11186623.0.0.4b3b1e4fWW6yji
var format string
switch opt.Image.Format {
case
imagePng,
imageJpg,
imageJpeg,
imageGif,
imageWebp:
format = opt.Image.Format
default:
opt.Image.Format = imageJpg
}
// https://oss-console-img-demo-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/example.jpg?x-oss-process=image/resize,h_100,m_lfit
process := "image/resize,m_lfit"
if opt.Image.Width > 0 {
process += ",w_" + strconv.Itoa(opt.Image.Width)
}
if opt.Image.Height > 0 {
process += ",h_" + strconv.Itoa(opt.Image.Height)
}
process += ",format," + format
opts = append(opts, oss.Process(process))
}
if !o.publicRead {
if opt.ContentType != "" {
opts = append(opts, oss.ResponseContentType(opt.ContentType))
}
if opt.Filename != "" {
opts = append(opts, oss.ResponseContentDisposition(`attachment; filename=`+strconv.Quote(opt.Filename)))
}
}
}
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
expire = time.Second
}
if !o.publicRead {
return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...)
}
rawParams, err := oss.GetRawParams(opts)
if err != nil {
return "", errs.Wrap(err, "AccessURL error")
}
params := getURLParams(*o.bucket.Client.Conn, rawParams)
return getURL(o.um, o.bucket.BucketName, name, params).String(), nil
}
func (o *OSS) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
// https://help.aliyun.com/zh/oss/developer-reference/postobject?spm=a2c4g.11186623.0.0.1cb83cebkP55nn
expires := time.Now().Add(duration)
conditions := []any{
map[string]string{"bucket": o.bucket.BucketName},
map[string]string{"key": name},
}
if size > 0 {
conditions = append(conditions, []any{"content-length-range", 0, size})
}
policy := map[string]any{
"expiration": expires.Format("2006-01-02T15:04:05.000Z"),
"conditions": conditions,
}
policyJson, err := json.Marshal(policy)
if err != nil {
return nil, errs.Wrap(err, "Marshal json error")
}
policyStr := base64.StdEncoding.EncodeToString(policyJson)
h := hmac.New(sha1.New, []byte(o.credentials.GetAccessKeySecret()))
if _, err := io.WriteString(h, policyStr); err != nil {
return nil, errs.Wrap(err, "WriteString error")
}
fd := &s3.FormData{
URL: o.bucketURL,
File: "file",
Expires: expires,
FormData: map[string]string{
"key": name,
"policy": policyStr,
"OSSAccessKeyId": o.credentials.GetAccessKeyID(),
"success_action_status": strconv.Itoa(successCode),
"signature": base64.StdEncoding.EncodeToString(h.Sum(nil)),
},
SuccessCodes: []int{successCode},
}
if contentType != "" {
fd.FormData["x-oss-content-type"] = contentType
}
return fd, nil
}
pkg/common/db/s3/s3.go
-166
View File
@@ -1,166 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package s3
import (
"context"
"net/http"
"net/url"
"time"
)
type PartLimit struct {
MinPartSize int64 `json:"minPartSize"`
MaxPartSize int64 `json:"maxPartSize"`
MaxNumSize int64 `json:"maxNumSize"`
}
type InitiateMultipartUploadResult struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
UploadID string `json:"uploadID"`
}
type MultipartUploadRequest struct {
UploadID string `json:"uploadId"`
Bucket string `json:"bucket"`
Key string `json:"key"`
Method string `json:"method"`
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
PartKey string `json:"partKey"`
PartSize int64 `json:"partSize"`
FirstPart int `json:"firstPart"`
}
type Part struct {
PartNumber int `json:"partNumber"`
ETag string `json:"etag"`
}
type CompleteMultipartUploadResult struct {
Location string `json:"location"`
Bucket string `json:"bucket"`
Key string `json:"key"`
ETag string `json:"etag"`
}
type SignResult struct {
Parts []SignPart `json:"parts"`
}
type ObjectInfo struct {
ETag string `json:"etag"`
Key string `json:"name"`
Size int64 `json:"size"`
LastModified time.Time `json:"lastModified"`
}
type CopyObjectInfo struct {
Key string `json:"name"`
ETag string `json:"etag"`
}
type FormData struct {
URL string `json:"url"`
File string `json:"file"`
Header http.Header `json:"header"`
FormData map[string]string `json:"form"`
Expires time.Time `json:"expires"`
SuccessCodes []int `json:"successActionStatus"`
}
type SignPart struct {
PartNumber int `json:"partNumber"`
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
}
type AuthSignResult struct {
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
Parts []SignPart `json:"parts"`
}
type InitiateUpload struct {
UploadID string `json:"uploadId"`
Bucket string `json:"bucket"`
Key string `json:"key"`
Method string `json:"method"`
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
PartKey string `json:"partKey"`
PartSize int64 `json:"partSize"`
FirstPart int `json:"firstPart"`
}
type UploadedPart struct {
PartNumber int `json:"partNumber"`
LastModified time.Time `json:"lastModified"`
ETag string `json:"etag"`
Size int64 `json:"size"`
}
type ListUploadedPartsResult struct {
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
NextPartNumberMarker int `xml:"NextPartNumberMarker"`
MaxParts int `xml:"MaxParts"`
UploadedParts []UploadedPart `xml:"Part"`
}
type Image struct {
Format string `json:"format"`
Width int `json:"width"`
Height int `json:"height"`
}
type AccessURLOption struct {
ContentType string `json:"contentType"`
Filename string `json:"filename"`
Image *Image `json:"image"`
}
type Interface interface {
Engine() string
PartLimit() *PartLimit
InitiateMultipartUpload(ctx context.Context, name string) (*InitiateMultipartUploadResult, error)
CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []Part) (*CompleteMultipartUploadResult, error)
PartSize(ctx context.Context, size int64) (int64, error)
AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*AuthSignResult, error)
PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error)
DeleteObject(ctx context.Context, name string) error
CopyObject(ctx context.Context, src string, dst string) (*CopyObjectInfo, error)
StatObject(ctx context.Context, name string) (*ObjectInfo, error)
IsNotFound(err error) bool
AbortMultipartUpload(ctx context.Context, uploadID string, name string) error
ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*ListUploadedPartsResult, error)
AccessURL(ctx context.Context, name string, expire time.Duration, opt *AccessURLOption) (string, error)
FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*FormData, error)
}
pkg/common/db/table/relation/black.go
+3 -3
View File
@@ -18,7 +18,7 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/tools/db/pagination"
)
type BlackModel struct {
@@ -33,8 +33,8 @@ type BlackModel struct {
type BlackModelInterface interface {
Create(ctx context.Context, blacks []*BlackModel) (err error)
Delete(ctx context.Context, blacks []*BlackModel) (err error)
//UpdateByMap(ctx context.Context, ownerUserID, blockUserID string, args map[string]any) (err error)
//Update(ctx context.Context, blacks []*BlackModel) (err error)
// UpdateByMap(ctx context.Context, ownerUserID, blockUserID string, args map[string]any) (err error)
// Update(ctx context.Context, blacks []*BlackModel) (err error)
Find(ctx context.Context, blacks []*BlackModel) (blackList []*BlackModel, err error)
Take(ctx context.Context, ownerUserID, blockUserID string) (black *BlackModel, err error)
FindOwnerBlacks(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (total int64, blacks []*BlackModel, err error)
pkg/common/db/table/relation/conversation.go
+1 -1
View File
@@ -18,7 +18,7 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/tools/db/pagination"
)
type ConversationModel struct {
pkg/common/db/s3/cont/doc.go → pkg/common/db/table/relation/doc.go
+2 -2
View File
@@ -1,4 +1,4 @@
// Copyright © 2023 OpenIM. All rights reserved.
// Copyright © 2024 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,4 +12,4 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package cont // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cont"
package relation // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
pkg/common/db/table/relation/friend.go
+1 -1
View File
@@ -18,7 +18,7 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/tools/db/pagination"
)
// FriendModel represents the data structure for a friend relationship in MongoDB.
pkg/common/db/table/relation/friend_request.go
+1 -1
View File
@@ -18,7 +18,7 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/tools/db/pagination"
)
type FriendRequestModel struct {
pkg/common/db/table/relation/group.go
+1 -1
View File
@@ -18,7 +18,7 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/tools/db/pagination"
)
type GroupModel struct {
pkg/common/db/table/relation/group_member.go
+6 -6
View File
@@ -18,7 +18,7 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/tools/db/pagination"
)
type GroupMemberModel struct {
@@ -36,10 +36,10 @@ type GroupMemberModel struct {
}
type GroupMemberModelInterface interface {
//NewTx(tx any) GroupMemberModelInterface
// NewTx(tx any) GroupMemberModelInterface
Create(ctx context.Context, groupMembers []*GroupMemberModel) (err error)
Delete(ctx context.Context, groupID string, userIDs []string) (err error)
//DeleteGroup(ctx context.Context, groupIDs []string) (err error)
// DeleteGroup(ctx context.Context, groupIDs []string) (err error)
Update(ctx context.Context, groupID string, userID string, data map[string]any) (err error)
UpdateRoleLevel(ctx context.Context, groupID string, userID string, roleLevel int32) error
FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error)
@@ -47,11 +47,11 @@ type GroupMemberModelInterface interface {
TakeOwner(ctx context.Context, groupID string) (groupMember *GroupMemberModel, err error)
SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*GroupMemberModel, err error)
FindRoleLevelUserIDs(ctx context.Context, groupID string, roleLevel int32) ([]string, error)
//MapGroupMemberNum(ctx context.Context, groupIDs []string) (count map[string]uint32, err error)
//FindJoinUserID(ctx context.Context, groupIDs []string) (groupUsers map[string][]string, err error)
// MapGroupMemberNum(ctx context.Context, groupIDs []string) (count map[string]uint32, err error)
// FindJoinUserID(ctx context.Context, groupIDs []string) (groupUsers map[string][]string, err error)
FindUserJoinedGroupID(ctx context.Context, userID string) (groupIDs []string, err error)
TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error)
//FindUsersJoinedGroupID(ctx context.Context, userIDs []string) (map[string][]string, error)
// FindUsersJoinedGroupID(ctx context.Context, userIDs []string) (map[string][]string, error)
FindUserManagedGroupID(ctx context.Context, userID string) (groupIDs []string, err error)
IsUpdateRoleLevel(data map[string]any) bool
}
pkg/common/db/table/relation/group_request.go
+1 -1
View File
@@ -18,7 +18,7 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/tools/db/pagination"
)
type GroupRequestModel struct {
pkg/common/db/table/relation/log.go
+1 -1
View File
@@ -18,7 +18,7 @@ import (
"context"
"time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/tools/db/pagination"
)
type LogModel struct {
pkg/common/db/table/unrelation/msg.go → pkg/common/db/table/relation/msg.go
+16 -22
View File
@@ -12,15 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package unrelation
package relation
import (
"context"
"strconv"
"time"
"github.com/OpenIMSDK/protocol/msg"
"github.com/OpenIMSDK/protocol/sdkws"
"github.com/openimsdk/protocol/msg"
"github.com/openimsdk/protocol/sdkws"
"github.com/openimsdk/tools/errs"
"go.mongodb.org/mongo-driver/mongo"
)
@@ -32,6 +33,8 @@ const (
NewestList = -1
)
var ErrMsgListNotExist = errs.New("user not have msg in mongoDB")
type MsgDocModel struct {
DocID string `bson:"doc_id"`
Msg []*MsgInfoModel `bson:"msgs"`
@@ -110,24 +113,15 @@ type MsgDocModelInterface interface {
DeleteMsgsInOneDocByIndex(ctx context.Context, docID string, indexes []int) error
MarkSingleChatMsgsAsRead(ctx context.Context, userID string, docID string, indexes []int64) error
SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*MsgInfoModel, error)
RangeUserSendCount(
ctx context.Context,
start time.Time,
end time.Time,
group bool,
ase bool,
pageNumber int32,
showNumber int32,
) (msgCount int64, userCount int64, users []*UserCount, dateCount map[string]int64, err error)
RangeGroupSendCount(
ctx context.Context,
start time.Time,
end time.Time,
ase bool,
pageNumber int32,
showNumber int32,
) (msgCount int64, userCount int64, groups []*GroupCount, dateCount map[string]int64, err error)
RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*UserCount, dateCount map[string]int64, err error)
RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*GroupCount, dateCount map[string]int64, err error)
ConvertMsgsDocLen(ctx context.Context, conversationIDs []string)
DeleteDoc(ctx context.Context, docID string) error
DeleteMsgByIndex(ctx context.Context, docID string, index []int) error
GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*MsgDocModel, error)
//ClearMsg(ctx context.Context, t time.Time) (int64, error)
}
func (MsgDocModel) TableName() string {
@@ -165,11 +159,11 @@ func (m MsgDocModel) GetDocIDSeqsMap(conversationID string, seqs []int64) map[st
return t
}
func (m MsgDocModel) GetMsgIndex(seq int64) int64 {
func (MsgDocModel) GetMsgIndex(seq int64) int64 {
return (seq - 1) % singleGocMsgNum
}
func (m MsgDocModel) indexGen(conversationID string, seqSuffix int64) string {
func (MsgDocModel) indexGen(conversationID string, seqSuffix int64) string {
return conversationID + ":" + strconv.FormatInt(seqSuffix, 10)
}
pkg/common/db/table/unrelation/user.go → pkg/common/db/table/relation/subscribe.go
+6 -6
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package unrelation
package relation
import "context"
@@ -21,18 +21,18 @@ const (
SubscribeUser = "subscribe_user"
)
// UserModel collection structure.
type UserModel struct {
// SubscribeUserModel collection structure.
type SubscribeUserModel struct {
UserID string `bson:"user_id" json:"userID"`
UserIDList []string `bson:"user_id_list" json:"userIDList"`
}
func (UserModel) TableName() string {
func (SubscribeUserModel) TableName() string {
return SubscribeUser
}
// UserModelInterface Operation interface of user mongodb.
type UserModelInterface interface {
// SubscribeUserModelInterface Operation interface of user mongodb.
type SubscribeUserModelInterface interface {
// AddSubscriptionList Subscriber's handling of thresholds.
AddSubscriptionList(ctx context.Context, userID string, userIDList []string) error
// UnsubscriptionList Handling of unsubscribe.
pkg/common/db/table/relation/user.go
+3 -3
View File
@@ -18,8 +18,8 @@ import (
"context"
"time"
"github.com/OpenIMSDK/protocol/user"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/protocol/user"
"github.com/openimsdk/tools/db/pagination"
)
type UserModel struct {
@@ -65,7 +65,7 @@ type UserModelInterface interface {
CountTotal(ctx context.Context, before *time.Time) (count int64, err error)
// Get user total quantity every day
CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
//CRUD user command
// CRUD user command
AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error
DeleteUserCommand(ctx context.Context, userID string, Type int32, UUID string) error
UpdateUserCommand(ctx context.Context, userID string, Type int32, UUID string, val map[string]any) error
pkg/common/db/table/relation/utils.go
+2 -2
View File
@@ -15,7 +15,7 @@
package relation
import (
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/tools/errs"
"go.mongodb.org/mongo-driver/mongo"
)
@@ -31,5 +31,5 @@ type GroupSimpleUserID struct {
}
func IsNotFound(err error) bool {
return utils.Unwrap(err) == mongo.ErrNoDocuments
return errs.Unwrap(err) == mongo.ErrNoDocuments
}
pkg/common/db/table/unrelation/common.go
-20
View File
@@ -1,20 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package unrelation
type CommonUserModel struct {
UserID string `bson:"user_id"`
UserName string `bson:"user_name"`
}
pkg/common/db/table/unrelation/doc.go
-15
View File
@@ -1,15 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package unrelation // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
pkg/common/db/table/unrelation/super_group.go
-53
View File
@@ -1,53 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package unrelation
//import (
// "context"
//)
//
//const (
// CSuperGroup = "super_group"
// CUserToSuperGroup = "user_to_super_group"
//)
//
//type SuperGroupModel struct {
// GroupID string `bson:"group_id" json:"groupID"`
// MemberIDs []string `bson:"member_id_list" json:"memberIDList"`
//}
//
//func (SuperGroupModel) TableName() string {
// return CSuperGroup
//}
//
//type UserToSuperGroupModel struct {
// UserID string `bson:"user_id" json:"userID"`
// GroupIDs []string `bson:"group_id_list" json:"groupIDList"`
//}
//
//func (UserToSuperGroupModel) TableName() string {
// return CUserToSuperGroup
//}
//
//type SuperGroupModelInterface interface {
// CreateSuperGroup(ctx context.Context, groupID string, initMemberIDs []string) error
// TakeSuperGroup(ctx context.Context, groupID string) (group *SuperGroupModel, err error)
// FindSuperGroup(ctx context.Context, groupIDs []string) (groups []*SuperGroupModel, err error)
// AddUserToSuperGroup(ctx context.Context, groupID string, userIDs []string) error
// RemoverUserFromSuperGroup(ctx context.Context, groupID string, userIDs []string) error
// GetSuperGroupByUserID(ctx context.Context, userID string) (*UserToSuperGroupModel, error)
// DeleteSuperGroup(ctx context.Context, groupID string) error
// RemoveGroupFromUser(ctx context.Context, groupID string, userIDs []string) error
//}
pkg/common/db/unrelation/doc.go
-15
View File
@@ -1,15 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package unrelation // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
pkg/common/db/unrelation/mongo.go
-168
View File
@@ -1,168 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package unrelation
import (
"context"
"fmt"
"os"
"strings"
"time"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/mw/specialerror"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
const (
maxRetry = 10 // number of retries
mongoConnTimeout = 10 * time.Second
)
type Mongo struct {
db *mongo.Client
config *config.GlobalConfig
}
// NewMongo Initialize MongoDB connection.
func NewMongo(config *config.GlobalConfig) (*Mongo, error) {
specialerror.AddReplace(mongo.ErrNoDocuments, errs.ErrRecordNotFound)
uri := buildMongoURI(config)
var mongoClient *mongo.Client
var err error
// Retry connecting to MongoDB
for i := 0; i <= maxRetry; i++ {
ctx, cancel := context.WithTimeout(context.Background(), mongoConnTimeout)
defer cancel()
mongoClient, err = mongo.Connect(ctx, options.Client().ApplyURI(uri))
if err == nil {
if err = mongoClient.Ping(ctx, nil); err != nil {
return nil, errs.Wrap(err, uri)
}
return &Mongo{db: mongoClient, config: config}, nil
}
if shouldRetry(err) {
time.Sleep(time.Second) // exponential backoff could be implemented here
continue
}
}
return nil, errs.Wrap(err, uri)
}
func buildMongoURI(config *config.GlobalConfig) string {
uri := os.Getenv("MONGO_URI")
if uri != "" {
return uri
}
if config.Mongo.Uri != "" {
return config.Mongo.Uri
}
username := os.Getenv("MONGO_OPENIM_USERNAME")
password := os.Getenv("MONGO_OPENIM_PASSWORD")
address := os.Getenv("MONGO_ADDRESS")
port := os.Getenv("MONGO_PORT")
database := os.Getenv("MONGO_DATABASE")
maxPoolSize := os.Getenv("MONGO_MAX_POOL_SIZE")
if username == "" {
username = config.Mongo.Username
}
if password == "" {
password = config.Mongo.Password
}
if address == "" {
address = strings.Join(config.Mongo.Address, ",")
} else if port != "" {
address = fmt.Sprintf("%s:%s", address, port)
}
if database == "" {
database = config.Mongo.Database
}
if maxPoolSize == "" {
maxPoolSize = fmt.Sprint(config.Mongo.MaxPoolSize)
}
if username != "" && password != "" {
return fmt.Sprintf("mongodb://%s:%s@%s/%s?maxPoolSize=%s", username, password, address, database, maxPoolSize)
}
return fmt.Sprintf("mongodb://%s/%s?maxPoolSize=%s", address, database, maxPoolSize)
}
func shouldRetry(err error) bool {
if cmdErr, ok := err.(mongo.CommandError); ok {
return cmdErr.Code != 13 && cmdErr.Code != 18
}
return true
}
// GetClient returns the MongoDB client.
func (m *Mongo) GetClient() *mongo.Client {
return m.db
}
// GetDatabase returns the specific database from MongoDB.
func (m *Mongo) GetDatabase(database string) *mongo.Database {
return m.db.Database(database)
}
// CreateMsgIndex creates an index for messages in MongoDB.
func (m *Mongo) CreateMsgIndex() error {
return m.createMongoIndex(unrelation.Msg, true, "doc_id")
}
// createMongoIndex creates an index in a MongoDB collection.
func (m *Mongo) createMongoIndex(collection string, isUnique bool, keys ...string) error {
db := m.GetDatabase(m.config.Mongo.Database).Collection(collection)
opts := options.CreateIndexes().SetMaxTime(10 * time.Second)
indexView := db.Indexes()
keysDoc := buildIndexKeys(keys)
index := mongo.IndexModel{
Keys: keysDoc,
}
if isUnique {
index.Options = options.Index().SetUnique(true)
}
_, err := indexView.CreateOne(context.Background(), index, opts)
if err != nil {
return errs.Wrap(err, "CreateIndex")
}
return nil
}
// buildIndexKeys builds the BSON document for index keys.
func buildIndexKeys(keys []string) bson.D {
keysDoc := bson.D{}
for _, key := range keys {
direction := 1 // default direction is ascending
if strings.HasPrefix(key, "-") {
direction = -1 // descending order for prefixed with "-"
key = strings.TrimLeft(key, "-")
}
keysDoc = append(keysDoc, bson.E{Key: key, Value: direction})
}
return keysDoc
}
pkg/common/db/unrelation/msg_convert.go
-81
View File
@@ -1,81 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package unrelation
import (
"context"
"fmt"
"github.com/OpenIMSDK/tools/log"
table "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func (m *MsgMongoDriver) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) {
for _, conversationID := range conversationIDs {
regex := primitive.Regex{Pattern: fmt.Sprintf("^%s:", conversationID)}
cursor, err := m.MsgCollection.Find(ctx, bson.M{"doc_id": regex})
if err != nil {
log.ZError(ctx, "convertAll find msg doc failed", err, "conversationID", conversationID)
continue
}
var msgDocs []table.MsgDocModel
err = cursor.All(ctx, &msgDocs)
if err != nil {
log.ZError(ctx, "convertAll cursor all failed", err, "conversationID", conversationID)
continue
}
if len(msgDocs) < 1 {
continue
}
log.ZInfo(ctx, "msg doc convert", "conversationID", conversationID, "len(msgDocs)", len(msgDocs))
if len(msgDocs[0].Msg) == int(m.model.GetSingleGocMsgNum5000()) {
if _, err := m.MsgCollection.DeleteMany(ctx, bson.M{"doc_id": regex}); err != nil {
log.ZError(ctx, "convertAll delete many failed", err, "conversationID", conversationID)
continue
}
var newMsgDocs []any
for _, msgDoc := range msgDocs {
if int64(len(msgDoc.Msg)) == m.model.GetSingleGocMsgNum() {
continue
}
var index int64
for index < int64(len(msgDoc.Msg)) {
msg := msgDoc.Msg[index]
if msg != nil && msg.Msg != nil {
msgDocModel := table.MsgDocModel{DocID: m.model.GetDocID(conversationID, msg.Msg.Seq)}
end := index + m.model.GetSingleGocMsgNum()
if int(end) >= len(msgDoc.Msg) {
msgDocModel.Msg = msgDoc.Msg[index:]
} else {
msgDocModel.Msg = msgDoc.Msg[index:end]
}
newMsgDocs = append(newMsgDocs, msgDocModel)
index = end
} else {
break
}
}
}
_, err = m.MsgCollection.InsertMany(ctx, newMsgDocs)
if err != nil {
log.ZError(ctx, "convertAll insert many failed", err, "conversationID", conversationID, "len(newMsgDocs)", len(newMsgDocs))
} else {
log.ZInfo(ctx, "msg doc convert", "conversationID", conversationID, "len(newMsgDocs)", len(newMsgDocs))
}
}
}
}
pkg/common/db/unrelation/super_group.go
-163
View File
@@ -1,163 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package unrelation
//
//import (
// "context"
//
// "go.mongodb.org/mongo-driver/bson"
// "go.mongodb.org/mongo-driver/mongo"
// "go.mongodb.org/mongo-driver/mongo/options"
//
// "github.com/OpenIMSDK/tools/utils"
//
// "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
//)
//
//func NewSuperGroupMongoDriver(database *mongo.Database) unrelation.SuperGroupModelInterface {
// return &SuperGroupMongoDriver{
// superGroupCollection: database.Collection(unrelation.CSuperGroup),
// userToSuperGroupCollection: database.Collection(unrelation.CUserToSuperGroup),
// }
//}
//
//type SuperGroupMongoDriver struct {
// superGroupCollection *mongo.Collection
// userToSuperGroupCollection *mongo.Collection
//}
//
//func (s *SuperGroupMongoDriver) CreateSuperGroup(ctx context.Context, groupID string, initMemberIDs []string) error {
// _, err := s.superGroupCollection.InsertOne(ctx, &unrelation.SuperGroupModel{
// GroupID: groupID,
// MemberIDs: initMemberIDs,
// })
// if err != nil {
// return err
// }
// for _, userID := range initMemberIDs {
// _, err = s.userToSuperGroupCollection.UpdateOne(
// ctx,
// bson.M{"user_id": userID},
// bson.M{"$addToSet": bson.M{"group_id_list": groupID}},
// &options.UpdateOptions{
// Upsert: utils.ToPtr(true),
// },
// )
// if err != nil {
// return err
// }
// }
// return nil
//}
//
//func (s *SuperGroupMongoDriver) TakeSuperGroup(
// ctx context.Context,
// groupID string,
//) (group *unrelation.SuperGroupModel, err error) {
// if err := s.superGroupCollection.FindOne(ctx, bson.M{"group_id": groupID}).Decode(&group); err != nil {
// return nil, utils.Wrap(err, "")
// }
// return group, nil
//}
//
//func (s *SuperGroupMongoDriver) FindSuperGroup(
// ctx context.Context,
// groupIDs []string,
//) (groups []*unrelation.SuperGroupModel, err error) {
// cursor, err := s.superGroupCollection.Find(ctx, bson.M{"group_id": bson.M{
// "$in": groupIDs,
// }})
// if err != nil {
// return nil, err
// }
// defer cursor.Close(ctx)
// if err := cursor.All(ctx, &groups); err != nil {
// return nil, utils.Wrap(err, "")
// }
// return groups, nil
//}
//
//func (s *SuperGroupMongoDriver) AddUserToSuperGroup(ctx context.Context, groupID string, userIDs []string) error {
// _, err := s.superGroupCollection.UpdateOne(
// ctx,
// bson.M{"group_id": groupID},
// bson.M{"$addToSet": bson.M{"member_id_list": bson.M{"$each": userIDs}}},
// )
// if err != nil {
// return err
// }
// upsert := true
// opts := &options.UpdateOptions{
// Upsert: &upsert,
// }
// for _, userID := range userIDs {
// _, err = s.userToSuperGroupCollection.UpdateOne(
// ctx,
// bson.M{"user_id": userID},
// bson.M{"$addToSet": bson.M{"group_id_list": groupID}},
// opts,
// )
// if err != nil {
// return utils.Wrap(err, "transaction failed")
// }
// }
// return nil
//}
//
//func (s *SuperGroupMongoDriver) RemoverUserFromSuperGroup(ctx context.Context, groupID string, userIDs []string) error {
// _, err := s.superGroupCollection.UpdateOne(
// ctx,
// bson.M{"group_id": groupID},
// bson.M{"$pull": bson.M{"member_id_list": bson.M{"$in": userIDs}}},
// )
// if err != nil {
// return err
// }
// err = s.RemoveGroupFromUser(ctx, groupID, userIDs)
// if err != nil {
// return err
// }
// return nil
//}
//
//func (s *SuperGroupMongoDriver) GetSuperGroupByUserID(
// ctx context.Context,
// userID string,
//) (*unrelation.UserToSuperGroupModel, error) {
// var user unrelation.UserToSuperGroupModel
// err := s.userToSuperGroupCollection.FindOne(ctx, bson.M{"user_id": userID}).Decode(&user)
// return &user, utils.Wrap(err, "")
//}
//
//func (s *SuperGroupMongoDriver) DeleteSuperGroup(ctx context.Context, groupID string) error {
// group, err := s.TakeSuperGroup(ctx, groupID)
// if err != nil {
// return err
// }
// if _, err := s.superGroupCollection.DeleteOne(ctx, bson.M{"group_id": groupID}); err != nil {
// return utils.Wrap(err, "")
// }
// return s.RemoveGroupFromUser(ctx, groupID, group.MemberIDs)
//}
//
//func (s *SuperGroupMongoDriver) RemoveGroupFromUser(ctx context.Context, groupID string, userIDs []string) error {
// _, err := s.userToSuperGroupCollection.UpdateOne(
// ctx,
// bson.M{"user_id": bson.M{"$in": userIDs}},
// bson.M{"$pull": bson.M{"group_id_list": groupID}},
// )
// return utils.Wrap(err, "")
//}