IM/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2026-05-15 06:25:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into add-new-feature-#484

Browse Source 
# Conflicts:
#	pkg/common/db/unrelation/extend_msg.go
This commit is contained in:
plutoyty
2023-07-12 19:16:48 +08:00
parent 35b1f75036 568e6a8e86
commit 9ac47e3423
154 changed files with 7800 additions and 10503 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/common/cmd/cron_task.go
+1 -1
View File
@@ -21,7 +21,7 @@ type CronTaskCmd struct {
}
func NewCronTaskCmd() *CronTaskCmd {
return &CronTaskCmd{NewRootCmd("cronTask")}
return &CronTaskCmd{NewRootCmd("cronTask", WithCronTaskLogName())}
}
func (c *CronTaskCmd) addRunE(f func() error) {
pkg/common/cmd/root.go
+28 -5
View File
@@ -31,17 +31,40 @@ type RootCmd struct {
prometheusPort int
}
func NewRootCmd(name string) (rootCmd *RootCmd) {
type CmdOpts struct {
loggerPrefixName string
}
func WithCronTaskLogName() func(*CmdOpts) {
return func(opts *CmdOpts) {
opts.loggerPrefixName = "OpenIM.CronTask.log.all"
}
}
func WithLogName(logName string) func(*CmdOpts) {
return func(opts *CmdOpts) {
opts.loggerPrefixName = logName
}
}
func NewRootCmd(name string, opts ...func(*CmdOpts)) (rootCmd *RootCmd) {
rootCmd = &RootCmd{Name: name}
c := cobra.Command{
Use: "start",
Short: fmt.Sprintf(`Start %s server`, name),
Long: fmt.Sprintf(`Start %s server`, name),
Use: "start openIM application",
Short: fmt.Sprintf(`Start %s `, name),
Long: fmt.Sprintf(`Start %s `, name),
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := rootCmd.getConfFromCmdAndInit(cmd); err != nil {
panic(err)
}
if err := log.InitFromConfig("OpenIM.log.all", name, config.Config.Log.RemainLogLevel, config.Config.Log.IsStdout, config.Config.Log.IsJson, config.Config.Log.StorageLocation, config.Config.Log.RemainRotationCount); err != nil {
cmdOpts := &CmdOpts{}
for _, opt := range opts {
opt(cmdOpts)
}
if cmdOpts.loggerPrefixName == "" {
cmdOpts.loggerPrefixName = "OpenIM.log.all"
}
if err := log.InitFromConfig(cmdOpts.loggerPrefixName, name, config.Config.Log.RemainLogLevel, config.Config.Log.IsStdout, config.Config.Log.IsJson, config.Config.Log.StorageLocation, config.Config.Log.RemainRotationCount); err != nil {
panic(err)
}
return nil
pkg/common/cmd/rpc.go
+3 -21
View File
@@ -1,27 +1,12 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"errors"
"github.com/spf13/cobra"
"google.golang.org/grpc"
"github.com/OpenIMSDK/Open-IM-Server/pkg/discoveryregistry"
"github.com/OpenIMSDK/Open-IM-Server/pkg/startrpc"
"github.com/spf13/cobra"
"google.golang.org/grpc"
)
type RpcCmd struct {
@@ -41,10 +26,7 @@ func (a *RpcCmd) Exec() error {
return a.Execute()
}
func (a *RpcCmd) StartSvr(
name string,
rpcFn func(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error,
) error {
func (a *RpcCmd) StartSvr(name string, rpcFn func(discov discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error) error {
if a.GetPortFlag() == 0 {
return errors.New("port is required")
}
pkg/common/config/config.go
+22 -41
View File
@@ -91,15 +91,11 @@ type config struct {
MsgToPush struct {
Topic string `yaml:"topic"`
} `yaml:"msgToPush"`
MsgToModify struct {
Topic string `yaml:"topic"`
} `yaml:"msgToModify"`
ConsumerGroupID struct {
MsgToRedis string `yaml:"msgToRedis"`
MsgToMongo string `yaml:"msgToMongo"`
MsgToMySql string `yaml:"msgToMySql"`
MsgToPush string `yaml:"msgToPush"`
MsgToModify string `yaml:"msgToModify"`
MsgToRedis string `yaml:"msgToRedis"`
MsgToMongo string `yaml:"msgToMongo"`
MsgToMySql string `yaml:"msgToMySql"`
MsgToPush string `yaml:"msgToPush"`
} `yaml:"consumerGroupID"`
} `yaml:"kafka"`
@@ -117,42 +113,26 @@ type config struct {
Enable string `yaml:"enable"`
ApiURL string `yaml:"apiURL"`
Minio struct {
TempBucket string `yaml:"tempBucket"`
DataBucket string `yaml:"dataBucket"`
Location string `yaml:"location"`
Endpoint string `yaml:"endpoint"`
AccessKeyID string `yaml:"accessKeyID"`
SecretAccessKey string `yaml:"secretAccessKey"`
IsDistributedMod bool `yaml:"isDistributedMod"`
Bucket string `yaml:"bucket"`
Endpoint string `yaml:"endpoint"`
AccessKeyID string `yaml:"accessKeyID"`
SecretAccessKey string `yaml:"secretAccessKey"`
SessionToken string `yaml:"sessionToken"`
} `yaml:"minio"`
Tencent struct {
AppID string `yaml:"appID"`
Region string `yaml:"region"`
Bucket string `yaml:"bucket"`
SecretID string `yaml:"secretID"`
SecretKey string `yaml:"secretKey"`
} `yaml:"tencent"`
Ali struct {
RegionID string `yaml:"regionID"`
AccessKeyID string `yaml:"accessKeyID"`
AccessKeySecret string `yaml:"accessKeySecret"`
StsEndpoint string `yaml:"stsEndpoint"`
OssEndpoint string `yaml:"ossEndpoint"`
Bucket string `yaml:"bucket"`
FinalHost string `yaml:"finalHost"`
StsDurationSeconds int64 `yaml:"stsDurationSeconds"`
OssRoleArn string `yaml:"OssRoleArn"`
} `yaml:"ali"`
Aws struct {
Cos struct {
BucketURL string `yaml:"bucketURL"`
SecretID string `yaml:"secretID"`
SecretKey string `yaml:"secretKey"`
SessionToken string `yaml:"sessionToken"`
} `yaml:"cos"`
Oss struct {
Endpoint string `yaml:"endpoint"`
Bucket string `yaml:"bucket"`
BucketURL string `yaml:"bucketURL"`
AccessKeyID string `yaml:"accessKeyID"`
AccessKeySecret string `yaml:"accessKeySecret"`
Region string `yaml:"region"`
Bucket string `yaml:"bucket"`
FinalHost string `yaml:"finalHost"`
RoleArn string `yaml:"roleArn"`
ExternalId string `yaml:"externalId"`
RoleSessionName string `yaml:"roleSessionName"`
} `yaml:"aws"`
SessionToken string `yaml:"sessionToken"`
} `yaml:"oss"`
} `yaml:"object"`
RpcPort struct {
@@ -229,6 +209,7 @@ type config struct {
SingleMessageHasReadReceiptEnable bool `yaml:"singleMessageHasReadReceiptEnable"`
RetainChatRecords int `yaml:"retainChatRecords"`
ChatRecordsClearTime string `yaml:"chatRecordsClearTime"`
MsgDestructTime string `yaml:"msgDestructTime"`
Secret string `yaml:"secret"`
TokenPolicy struct {
Expire int64 `yaml:"expire"`
pkg/common/convert/conversation.go
+2
View File
@@ -22,6 +22,7 @@ import (
func ConversationDB2Pb(conversationDB *relation.ConversationModel) *conversation.Conversation {
conversationPB := &conversation.Conversation{}
conversationPB.LatestMsgDestructTime = conversationDB.LatestMsgDestructTime.Unix()
if err := utils.CopyStructFields(conversationPB, conversationDB); err != nil {
return nil
}
@@ -34,6 +35,7 @@ func ConversationsDB2Pb(conversationsDB []*relation.ConversationModel) (conversa
if err := utils.CopyStructFields(conversationPB, conversationDB); err != nil {
continue
}
conversationPB.LatestMsgDestructTime = conversationDB.LatestMsgDestructTime.Unix()
conversationsPB = append(conversationsPB, conversationPB)
}
return conversationsPB
pkg/common/db/cache/extend_msg_set.go
Vendored
-101
View File
@@ -1,101 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cache
import (
"context"
"time"
"github.com/dtm-labs/rockscache"
"github.com/redis/go-redis/v9"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
)
const (
extendMsgSetCache = "EXTEND_MSG_SET_CACHE:"
extendMsgCache = "EXTEND_MSG_CACHE:"
)
type ExtendMsgSetCache interface {
metaCache
NewCache() ExtendMsgSetCache
GetExtendMsg(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
firstModifyTime int64,
) (extendMsg *unrelation.ExtendMsgModel, err error)
DelExtendMsg(clientMsgID string) ExtendMsgSetCache
}
type ExtendMsgSetCacheRedis struct {
metaCache
expireTime time.Duration
rcClient *rockscache.Client
extendMsgSetDB unrelation.ExtendMsgSetModelInterface
}
func NewExtendMsgSetCacheRedis(
rdb redis.UniversalClient,
extendMsgSetDB unrelation.ExtendMsgSetModelInterface,
options rockscache.Options,
) ExtendMsgSetCache {
rcClient := rockscache.NewClient(rdb, options)
return &ExtendMsgSetCacheRedis{
metaCache: NewMetaCacheRedis(rcClient),
expireTime: time.Second * 30 * 60,
extendMsgSetDB: extendMsgSetDB,
rcClient: rcClient,
}
}
func (e *ExtendMsgSetCacheRedis) NewCache() ExtendMsgSetCache {
return &ExtendMsgSetCacheRedis{
metaCache: NewMetaCacheRedis(e.rcClient, e.metaCache.GetPreDelKeys()...),
expireTime: e.expireTime,
extendMsgSetDB: e.extendMsgSetDB,
rcClient: e.rcClient,
}
}
func (e *ExtendMsgSetCacheRedis) getKey(clientMsgID string) string {
return extendMsgCache + clientMsgID
}
func (e *ExtendMsgSetCacheRedis) GetExtendMsg(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
firstModifyTime int64,
) (extendMsg *unrelation.ExtendMsgModel, err error) {
return getCache(
ctx,
e.rcClient,
e.getKey(clientMsgID),
e.expireTime,
func(ctx context.Context) (*unrelation.ExtendMsgModel, error) {
return e.extendMsgSetDB.TakeExtendMsg(ctx, conversationID, sessionType, clientMsgID, firstModifyTime)
},
)
}
func (e *ExtendMsgSetCacheRedis) DelExtendMsg(clientMsgID string) ExtendMsgSetCache {
new := e.NewCache()
new.AddKeys(e.getKey(clientMsgID))
return new
}
pkg/common/db/controller/conversation.go
+34 -103
View File
@@ -1,21 +1,8 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controller
import (
"context"
"time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
@@ -27,22 +14,13 @@ import (
type ConversationDatabase interface {
//UpdateUserConversationFiled 更新用户该会话的属性信息
UpdateUsersConversationFiled(
ctx context.Context,
userIDs []string,
conversationID string,
args map[string]interface{},
) error
UpdateUsersConversationFiled(ctx context.Context, userIDs []string, conversationID string, args map[string]interface{}) error
//CreateConversation 创建一批新的会话
CreateConversation(ctx context.Context, conversations []*relationTb.ConversationModel) error
//SyncPeerUserPrivateConversation 同步对端私聊会话内部保证事务操作
SyncPeerUserPrivateConversationTx(ctx context.Context, conversation []*relationTb.ConversationModel) error
//FindConversations 根据会话ID获取某个用户的多个会话
FindConversations(
ctx context.Context,
ownerUserID string,
conversationIDs []string,
) ([]*relationTb.ConversationModel, error)
FindConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*relationTb.ConversationModel, error)
//FindRecvMsgNotNotifyUserIDs 获取超级大群开启免打扰的用户ID
FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error)
//GetUserAllConversation 获取一个用户在服务器上所有的会话
@@ -50,28 +28,17 @@ type ConversationDatabase interface {
//SetUserConversations 设置用户多个会话属性，如果会话不存在则创建，否则更新,内部保证原子性
SetUserConversations(ctx context.Context, ownerUserID string, conversations []*relationTb.ConversationModel) error
//SetUsersConversationFiledTx 设置多个用户会话关于某个字段的更新操作，如果会话不存在则创建，否则更新，内部保证事务操作
SetUsersConversationFiledTx(
ctx context.Context,
userIDs []string,
conversation *relationTb.ConversationModel,
filedMap map[string]interface{},
) error
SetUsersConversationFiledTx(ctx context.Context, userIDs []string, conversation *relationTb.ConversationModel, filedMap map[string]interface{}) error
CreateGroupChatConversation(ctx context.Context, groupID string, userIDs []string) error
GetConversationIDs(ctx context.Context, userID string) ([]string, error)
GetUserConversationIDsHash(ctx context.Context, ownerUserID string) (hash uint64, err error)
GetAllConversationIDs(ctx context.Context) ([]string, error)
GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error)
GetConversationsByConversationID(
ctx context.Context,
conversationIDs []string,
) ([]*relationTb.ConversationModel, error)
GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*relationTb.ConversationModel, error)
GetConversationIDsNeedDestruct(ctx context.Context) ([]*relationTb.ConversationModel, error)
}
func NewConversationDatabase(
conversation relationTb.ConversationModelInterface,
cache cache.ConversationCache,
tx tx.Tx,
) ConversationDatabase {
func NewConversationDatabase(conversation relationTb.ConversationModelInterface, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase {
return &conversationDatabase{
conversationDB: conversation,
cache: cache,
@@ -85,12 +52,7 @@ type conversationDatabase struct {
tx tx.Tx
}
func (c *conversationDatabase) SetUsersConversationFiledTx(
ctx context.Context,
userIDs []string,
conversation *relationTb.ConversationModel,
filedMap map[string]interface{},
) (err error) {
func (c *conversationDatabase) SetUsersConversationFiledTx(ctx context.Context, userIDs []string, conversation *relationTb.ConversationModel, filedMap map[string]interface{}) (err error) {
cache := c.cache.NewCache()
if err := c.tx.Transaction(func(tx any) error {
conversationTx := c.conversationDB.NewTx(tx)
@@ -113,12 +75,14 @@ func (c *conversationDatabase) SetUsersConversationFiledTx(
NotUserIDs := utils.DifferenceString(haveUserIDs, userIDs)
log.ZDebug(ctx, "SetUsersConversationFiledTx", "NotUserIDs", NotUserIDs, "haveUserIDs", haveUserIDs, "userIDs", userIDs)
var conversations []*relationTb.ConversationModel
now := time.Now()
for _, v := range NotUserIDs {
temp := new(relationTb.ConversationModel)
if err := utils.CopyStructFields(temp, conversation); err != nil {
return err
}
temp.OwnerUserID = v
temp.CreateTime = now
conversations = append(conversations, temp)
}
@@ -136,12 +100,7 @@ func (c *conversationDatabase) SetUsersConversationFiledTx(
return cache.ExecDel(ctx)
}
func (c *conversationDatabase) UpdateUsersConversationFiled(
ctx context.Context,
userIDs []string,
conversationID string,
args map[string]interface{},
) error {
func (c *conversationDatabase) UpdateUsersConversationFiled(ctx context.Context, userIDs []string, conversationID string, args map[string]interface{}) error {
_, err := c.conversationDB.UpdateByMap(ctx, userIDs, conversationID, args)
if err != nil {
return err
@@ -149,10 +108,7 @@ func (c *conversationDatabase) UpdateUsersConversationFiled(
return c.cache.DelUsersConversation(conversationID, userIDs...).ExecDel(ctx)
}
func (c *conversationDatabase) CreateConversation(
ctx context.Context,
conversations []*relationTb.ConversationModel,
) error {
func (c *conversationDatabase) CreateConversation(ctx context.Context, conversations []*relationTb.ConversationModel) error {
if err := c.conversationDB.Create(ctx, conversations); err != nil {
return err
}
@@ -165,35 +121,34 @@ func (c *conversationDatabase) CreateConversation(
return cache.DelConversationIDs(userIDs...).DelUserConversationIDsHash(userIDs...).ExecDel(ctx)
}
func (c *conversationDatabase) SyncPeerUserPrivateConversationTx(
ctx context.Context,
conversations []*relationTb.ConversationModel,
) error {
func (c *conversationDatabase) SyncPeerUserPrivateConversationTx(ctx context.Context, conversations []*relationTb.ConversationModel) error {
cache := c.cache.NewCache()
if err := c.tx.Transaction(func(tx any) error {
conversationTx := c.conversationDB.NewTx(tx)
for _, conversation := range conversations {
for _, v := range [][2]string{{conversation.OwnerUserID, conversation.UserID}, {conversation.UserID, conversation.OwnerUserID}} {
haveUserIDs, err := conversationTx.FindUserID(ctx, []string{v[0]}, []string{conversation.ConversationID})
ownerUserID := v[0]
userID := v[1]
haveUserIDs, err := conversationTx.FindUserID(ctx, []string{ownerUserID}, []string{conversation.ConversationID})
if err != nil {
return err
}
if len(haveUserIDs) > 0 {
_, err := conversationTx.UpdateByMap(ctx, []string{v[0]}, conversation.ConversationID, map[string]interface{}{"is_private_chat": conversation.IsPrivateChat})
_, err := conversationTx.UpdateByMap(ctx, []string{ownerUserID}, conversation.ConversationID, map[string]interface{}{"is_private_chat": conversation.IsPrivateChat})
if err != nil {
return err
}
cache = cache.DelUsersConversation(conversation.ConversationID, v[0])
cache = cache.DelUsersConversation(conversation.ConversationID, ownerUserID)
} else {
newConversation := *conversation
newConversation.OwnerUserID = v[0]
newConversation.UserID = v[1]
newConversation.OwnerUserID = ownerUserID
newConversation.UserID = userID
newConversation.ConversationID = conversation.ConversationID
newConversation.IsPrivateChat = conversation.IsPrivateChat
if err := conversationTx.Create(ctx, []*relationTb.ConversationModel{&newConversation}); err != nil {
return err
}
cache = cache.DelConversationIDs(v[0]).DelUserConversationIDsHash(v[0])
cache = cache.DelConversationIDs(ownerUserID).DelUserConversationIDsHash(ownerUserID)
}
}
}
@@ -201,37 +156,22 @@ func (c *conversationDatabase) SyncPeerUserPrivateConversationTx(
}); err != nil {
return err
}
return c.cache.ExecDel(ctx)
return cache.ExecDel(ctx)
}
func (c *conversationDatabase) FindConversations(
ctx context.Context,
ownerUserID string,
conversationIDs []string,
) ([]*relationTb.ConversationModel, error) {
func (c *conversationDatabase) FindConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*relationTb.ConversationModel, error) {
return c.cache.GetConversations(ctx, ownerUserID, conversationIDs)
}
func (c *conversationDatabase) GetConversation(
ctx context.Context,
ownerUserID string,
conversationID string,
) (*relationTb.ConversationModel, error) {
func (c *conversationDatabase) GetConversation(ctx context.Context, ownerUserID string, conversationID string) (*relationTb.ConversationModel, error) {
return c.cache.GetConversation(ctx, ownerUserID, conversationID)
}
func (c *conversationDatabase) GetUserAllConversation(
ctx context.Context,
ownerUserID string,
) ([]*relationTb.ConversationModel, error) {
func (c *conversationDatabase) GetUserAllConversation(ctx context.Context, ownerUserID string) ([]*relationTb.ConversationModel, error) {
return c.cache.GetUserAllConversations(ctx, ownerUserID)
}
func (c *conversationDatabase) SetUserConversations(
ctx context.Context,
ownerUserID string,
conversations []*relationTb.ConversationModel,
) error {
func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUserID string, conversations []*relationTb.ConversationModel) error {
cache := c.cache.NewCache()
if err := c.tx.Transaction(func(tx any) error {
var conversationIDs []string
@@ -281,11 +221,7 @@ func (c *conversationDatabase) FindRecvMsgNotNotifyUserIDs(ctx context.Context,
return c.cache.GetSuperGroupRecvMsgNotNotifyUserIDs(ctx, groupID)
}
func (c *conversationDatabase) CreateGroupChatConversation(
ctx context.Context,
groupID string,
userIDs []string,
) error {
func (c *conversationDatabase) CreateGroupChatConversation(ctx context.Context, groupID string, userIDs []string) error {
cache := c.cache.NewCache()
conversationID := utils.GetConversationIDBySessionType(constant.SuperGroupChatType, groupID)
if err := c.tx.Transaction(func(tx any) error {
@@ -325,10 +261,7 @@ func (c *conversationDatabase) GetConversationIDs(ctx context.Context, userID st
return c.cache.GetUserConversationIDs(ctx, userID)
}
func (c *conversationDatabase) GetUserConversationIDsHash(
ctx context.Context,
ownerUserID string,
) (hash uint64, err error) {
func (c *conversationDatabase) GetUserConversationIDsHash(ctx context.Context, ownerUserID string) (hash uint64, err error) {
return c.cache.GetUserConversationIDsHash(ctx, ownerUserID)
}
@@ -336,16 +269,14 @@ func (c *conversationDatabase) GetAllConversationIDs(ctx context.Context) ([]str
return c.conversationDB.GetAllConversationIDs(ctx)
}
func (c *conversationDatabase) GetUserAllHasReadSeqs(
ctx context.Context,
ownerUserID string,
) (map[string]int64, error) {
func (c *conversationDatabase) GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error) {
return c.cache.GetUserAllHasReadSeqs(ctx, ownerUserID)
}
func (c *conversationDatabase) GetConversationsByConversationID(
ctx context.Context,
conversationIDs []string,
) ([]*relationTb.ConversationModel, error) {
func (c *conversationDatabase) GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*relationTb.ConversationModel, error) {
return c.conversationDB.GetConversationsByConversationID(ctx, conversationIDs)
}
func (c *conversationDatabase) GetConversationIDsNeedDestruct(ctx context.Context) ([]*relationTb.ConversationModel, error) {
return c.conversationDB.GetConversationIDsNeedDestruct(ctx)
}
pkg/common/db/controller/extend_msg.go
-158
View File
@@ -1,158 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controller
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
unRelationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/tx"
)
// for mongoDB
type ExtendMsgDatabase interface {
CreateExtendMsgSet(ctx context.Context, set *unRelationTb.ExtendMsgSetModel) error
GetAllExtendMsgSet(
ctx context.Context,
ID string,
opts *unRelationTb.GetAllExtendMsgSetOpts,
) (sets []*unRelationTb.ExtendMsgSetModel, err error)
GetExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
maxMsgUpdateTime int64,
) (*unRelationTb.ExtendMsgSetModel, error)
InsertExtendMsg(
ctx context.Context,
conversationID string,
sessionType int32,
msg *unRelationTb.ExtendMsgModel,
) error
InsertOrUpdateReactionExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
msgFirstModifyTime int64,
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
) error
DeleteReactionExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
msgFirstModifyTime int64,
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
) error
GetExtendMsg(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
maxMsgUpdateTime int64,
) (extendMsg *unRelationTb.ExtendMsgModel, err error)
}
type extendMsgDatabase struct {
database unRelationTb.ExtendMsgSetModelInterface
cache cache.ExtendMsgSetCache
ctxTx tx.CtxTx
}
func NewExtendMsgDatabase(
extendMsgModel unRelationTb.ExtendMsgSetModelInterface,
cache cache.ExtendMsgSetCache,
ctxTx tx.CtxTx,
) ExtendMsgDatabase {
return &extendMsgDatabase{database: extendMsgModel, cache: cache, ctxTx: ctxTx}
}
func (e *extendMsgDatabase) CreateExtendMsgSet(ctx context.Context, set *unRelationTb.ExtendMsgSetModel) error {
return e.database.CreateExtendMsgSet(ctx, set)
}
func (e *extendMsgDatabase) GetAllExtendMsgSet(
ctx context.Context,
conversationID string,
opts *unRelationTb.GetAllExtendMsgSetOpts,
) (sets []*unRelationTb.ExtendMsgSetModel, err error) {
return e.database.GetAllExtendMsgSet(ctx, conversationID, opts)
}
func (e *extendMsgDatabase) GetExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
maxMsgUpdateTime int64,
) (*unRelationTb.ExtendMsgSetModel, error) {
return e.database.GetExtendMsgSet(ctx, conversationID, sessionType, maxMsgUpdateTime)
}
func (e *extendMsgDatabase) InsertExtendMsg(
ctx context.Context,
conversationID string,
sessionType int32,
msg *unRelationTb.ExtendMsgModel,
) error {
return e.database.InsertExtendMsg(ctx, conversationID, sessionType, msg)
}
func (e *extendMsgDatabase) InsertOrUpdateReactionExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
msgFirstModifyTime int64,
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
) error {
return e.database.InsertOrUpdateReactionExtendMsgSet(
ctx,
conversationID,
sessionType,
clientMsgID,
msgFirstModifyTime,
reactionExtensionList,
)
}
func (e *extendMsgDatabase) DeleteReactionExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
msgFirstModifyTime int64,
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
) error {
return e.database.DeleteReactionExtendMsgSet(
ctx,
conversationID,
sessionType,
clientMsgID,
msgFirstModifyTime,
reactionExtensionList,
)
}
func (e *extendMsgDatabase) GetExtendMsg(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
maxMsgUpdateTime int64,
) (extendMsg *unRelationTb.ExtendMsgModel, err error) {
return e.cache.GetExtendMsg(ctx, conversationID, sessionType, clientMsgID, maxMsgUpdateTime)
}
pkg/common/db/controller/group.go
+14
View File
@@ -17,6 +17,7 @@ package controller
import (
"context"
"fmt"
"time"
"github.com/dtm-labs/rockscache"
"github.com/redis/go-redis/v9"
@@ -121,6 +122,11 @@ type GroupDatabase interface {
DeleteSuperGroup(ctx context.Context, groupID string) error
DeleteSuperGroupMember(ctx context.Context, groupID string, userIDs []string) error
CreateSuperGroupMember(ctx context.Context, groupID string, userIDs []string) error
// 获取群总数
CountTotal(ctx context.Context, before *time.Time) (count int64, err error)
// 获取范围内群增量
CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
}
func NewGroupDatabase(
@@ -562,3 +568,11 @@ func (g *groupDatabase) CreateSuperGroupMember(ctx context.Context, groupID stri
}
return g.cache.DelSuperGroupMemberIDs(groupID).DelJoinedSuperGroupIDs(userIDs...).ExecDel(ctx)
}
func (g *groupDatabase) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) {
return g.groupDB.CountTotal(ctx, before)
}
func (g *groupDatabase) CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) {
return g.groupDB.CountRangeEverydayTotal(ctx, start, end)
}
pkg/common/db/controller/msg.go
+110 -510
View File
@@ -1,21 +1,6 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controller
import (
"fmt"
"time"
"github.com/redis/go-redis/v9"
@@ -33,11 +18,10 @@ import (
"context"
"errors"
"go.mongodb.org/mongo-driver/mongo"
pbMsg "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"go.mongodb.org/mongo-driver/mongo"
)
const (
@@ -56,28 +40,17 @@ type CommonMsgDatabase interface {
DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error
DelUserDeleteMsgsList(ctx context.Context, conversationID string, seqs []int64)
// incrSeq然后批量插入缓存
BatchInsertChat2Cache(
ctx context.Context,
conversationID string,
msgs []*sdkws.MsgData,
) (seq int64, isNewConversation bool, err error)
BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, err error)
// 通过seqList获取mongo中写扩散消息
GetMsgBySeqsRange(
ctx context.Context,
userID string,
conversationID string,
begin, end, num, userMaxSeq int64,
) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error)
GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error)
// 通过seqList获取大群在 mongo里面的消息
GetMsgBySeqs(
ctx context.Context,
userID string,
conversationID string,
seqs []int64,
) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error)
GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error)
// 删除会话消息重置最小seq， remainTime为消息保留的时间单位秒,超时消息删除， 传0删除所有消息(此方法不删除redis cache)
DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error
// 用户标记删除过期消息返回标记删除的seq列表
UserMsgsDestruct(cte context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error)
// 用户根据seq删除消息
DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error
// 物理删除消息置空
@@ -101,11 +74,8 @@ type CommonMsgDatabase interface {
GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error)
UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error
GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (maxSeq, minSeq int64, err error)
GetConversationMinMaxSeqInMongoAndCache(
ctx context.Context,
conversationID string,
) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error)
GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error)
GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error)
SetSendMsgStatus(ctx context.Context, id string, status int32) error
GetSendMsgStatus(ctx context.Context, id string) (int32, error)
@@ -115,51 +85,17 @@ type CommonMsgDatabase interface {
MsgToPushMQ(ctx context.Context, key, conversarionID string, msg2mq *sdkws.MsgData) (int32, int64, error)
MsgToMongoMQ(ctx context.Context, key, conversarionID string, msgs []*sdkws.MsgData, lastSeq int64) error
// modify
JudgeMessageReactionExist(ctx context.Context, clientMsgID string, sessionType int32) (bool, error)
SetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey, value string) error
SetMessageReactionExpire(
ctx context.Context,
clientMsgID string,
sessionType int32,
expiration time.Duration,
) (bool, error)
GetExtendMsg(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
maxMsgUpdateTime int64,
) (*pbMsg.ExtendMsg, error)
InsertOrUpdateReactionExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
msgFirstModifyTime int64,
reactionExtensionList map[string]*sdkws.KeyValue,
) error
GetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey string) (string, error)
GetOneMessageAllReactionList(ctx context.Context, clientMsgID string, sessionType int32) (map[string]string, error)
DeleteOneMessageKey(ctx context.Context, clientMsgID string, sessionType int32, subKey string) error
DeleteReactionExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
msgFirstModifyTime int64,
reactionExtensionList map[string]*sdkws.KeyValue,
) error
RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*unRelationTb.UserCount, dateCount map[string]int64, err error)
RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*unRelationTb.GroupCount, dateCount map[string]int64, err error)
}
func NewCommonMsgDatabase(msgDocModel unRelationTb.MsgDocModelInterface, cacheModel cache.MsgModel) CommonMsgDatabase {
return &commonMsgDatabase{
msgDocDatabase: msgDocModel,
cache: cacheModel,
producer: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.LatestMsgToRedis.Topic),
producerToMongo: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToMongo.Topic),
producerToPush: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToPush.Topic),
producerToModify: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToModify.Topic),
msgDocDatabase: msgDocModel,
cache: cacheModel,
producer: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.LatestMsgToRedis.Topic),
producerToMongo: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToMongo.Topic),
producerToPush: kafka.NewKafkaProducer(config.Config.Kafka.Addr, config.Config.Kafka.MsgToPush.Topic),
}
}
@@ -171,15 +107,13 @@ func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database)
}
type commonMsgDatabase struct {
msgDocDatabase unRelationTb.MsgDocModelInterface
extendMsgDatabase unRelationTb.ExtendMsgSetModelInterface
extendMsgSetModel unRelationTb.ExtendMsgSetModel
msg unRelationTb.MsgDocModel
cache cache.MsgModel
producer *kafka.Producer
producerToMongo *kafka.Producer
producerToModify *kafka.Producer
producerToPush *kafka.Producer
msgDocDatabase unRelationTb.MsgDocModelInterface
msg unRelationTb.MsgDocModel
cache cache.MsgModel
producer *kafka.Producer
producerToMongo *kafka.Producer
producerToModify *kafka.Producer
producerToPush *kafka.Producer
}
func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error {
@@ -187,32 +121,16 @@ func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sd
return err
}
func (db *commonMsgDatabase) MsgToModifyMQ(
ctx context.Context,
key, conversationID string,
messages []*sdkws.MsgData,
) error {
func (db *commonMsgDatabase) MsgToModifyMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData) error {
if len(messages) > 0 {
_, _, err := db.producerToModify.SendMessage(
ctx,
key,
&pbMsg.MsgDataToModifyByMQ{ConversationID: conversationID, Messages: messages},
)
_, _, err := db.producerToModify.SendMessage(ctx, key, &pbMsg.MsgDataToModifyByMQ{ConversationID: conversationID, Messages: messages})
return err
}
return nil
}
func (db *commonMsgDatabase) MsgToPushMQ(
ctx context.Context,
key, conversationID string,
msg2mq *sdkws.MsgData,
) (int32, int64, error) {
partition, offset, err := db.producerToPush.SendMessage(
ctx,
key,
&pbMsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID},
)
func (db *commonMsgDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) {
partition, offset, err := db.producerToPush.SendMessage(ctx, key, &pbMsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID})
if err != nil {
log.ZError(ctx, "MsgToPushMQ", err, "key", key, "msg2mq", msg2mq)
return 0, 0, err
@@ -220,30 +138,15 @@ func (db *commonMsgDatabase) MsgToPushMQ(
return partition, offset, nil
}
func (db *commonMsgDatabase) MsgToMongoMQ(
ctx context.Context,
key, conversationID string,
messages []*sdkws.MsgData,
lastSeq int64,
) error {
func (db *commonMsgDatabase) MsgToMongoMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData, lastSeq int64) error {
if len(messages) > 0 {
_, _, err := db.producerToMongo.SendMessage(
ctx,
key,
&pbMsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages},
)
_, _, err := db.producerToMongo.SendMessage(ctx, key, &pbMsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages})
return err
}
return nil
}
func (db *commonMsgDatabase) BatchInsertBlock(
ctx context.Context,
conversationID string,
fields []any,
key int8,
firstSeq int64,
) error {
func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error {
if len(fields) == 0 {
return nil
}
@@ -344,12 +247,7 @@ func (db *commonMsgDatabase) BatchInsertBlock(
return nil
}
func (db *commonMsgDatabase) BatchInsertChat2DB(
ctx context.Context,
conversationID string,
msgList []*sdkws.MsgData,
currentMaxSeq int64,
) error {
func (db *commonMsgDatabase) BatchInsertChat2DB(ctx context.Context, conversationID string, msgList []*sdkws.MsgData, currentMaxSeq int64) error {
if len(msgList) == 0 {
return errs.ErrArgs.Wrap("msgList is empty")
}
@@ -395,21 +293,11 @@ func (db *commonMsgDatabase) BatchInsertChat2DB(
return db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq)
}
func (db *commonMsgDatabase) RevokeMsg(
ctx context.Context,
conversationID string,
seq int64,
revoke *unRelationTb.RevokeModel,
) error {
func (db *commonMsgDatabase) RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *unRelationTb.RevokeModel) error {
return db.BatchInsertBlock(ctx, conversationID, []any{revoke}, updateKeyRevoke, seq)
}
func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(
ctx context.Context,
userID string,
conversationID string,
totalSeqs []int64,
) error {
func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, totalSeqs []int64) error {
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, totalSeqs) {
var indexes []int64
for _, seq := range seqs {
@@ -432,11 +320,7 @@ func (db *commonMsgDatabase) DelUserDeleteMsgsList(ctx context.Context, conversa
db.cache.DelUserDeleteMsgsList(ctx, conversationID, seqs)
}
func (db *commonMsgDatabase) BatchInsertChat2Cache(
ctx context.Context,
conversationID string,
msgs []*sdkws.MsgData,
) (seq int64, isNew bool, err error) {
func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) {
currentMaxSeq, err := db.cache.GetMaxSeq(ctx, conversationID)
if err != nil && errs.Unwrap(err) != redis.Nil {
prome.Inc(prome.SeqGetFailedCounter)
@@ -483,11 +367,7 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(
return lastMaxSeq, isNew, utils.Wrap(err, "")
}
func (db *commonMsgDatabase) getMsgBySeqs(
ctx context.Context,
userID, conversationID string,
seqs []int64,
) (totalMsgs []*sdkws.MsgData, err error) {
func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversationID string, seqs []int64) (totalMsgs []*sdkws.MsgData, err error) {
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, seqs) {
//log.ZDebug(ctx, "getMsgBySeqs", "docID", docID, "seqs", seqs)
msgs, err := db.findMsgInfoBySeq(ctx, userID, docID, seqs)
@@ -501,50 +381,7 @@ func (db *commonMsgDatabase) getMsgBySeqs(
return totalMsgs, nil
}
// func (db *commonMsgDatabase) refetchDelSeqsMsgs(ctx context.Context, conversationID string, delNums, rangeBegin,
// begin int64) (seqMsgs []*unRelationTb.MsgDataModel, err error) {
// var reFetchSeqs []int64
// if delNums > 0 {
// newBeginSeq := rangeBegin - delNums
// if newBeginSeq >= begin {
// newEndSeq := rangeBegin - 1
// for i := newBeginSeq; i <= newEndSeq; i++ {
// reFetchSeqs = append(reFetchSeqs, i)
// }
// }
// }
// if len(reFetchSeqs) == 0 {
// return
// }
// if len(reFetchSeqs) > 0 {
// m := db.msg.GetDocIDSeqsMap(conversationID, reFetchSeqs)
// for docID, seqs := range m {
// msgs, _, err := db.findMsgInfoBySeq(ctx, docID, seqs)
// if err != nil {
// return nil, err
// }
// for _, msg := range msgs {
// if msg.Status != constant.MsgDeleted {
// seqMsgs = append(seqMsgs, msg)
// }
// }
// }
// }
// if len(seqMsgs) < int(delNums) {
// seqMsgs2, err := db.refetchDelSeqsMsgs(ctx, conversationID, delNums-int64(len(seqMsgs)), rangeBegin-1, begin)
// if err != nil {
// return seqMsgs, err
// }
// seqMsgs = append(seqMsgs, seqMsgs2...)
// }
// return seqMsgs, nil
// }
func (db *commonMsgDatabase) findMsgInfoBySeq(
ctx context.Context,
userID, docID string,
seqs []int64,
) (totalMsgs []*unRelationTb.MsgInfoModel, err error) {
func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, seqs []int64) (totalMsgs []*unRelationTb.MsgInfoModel, err error) {
msgs, err := db.msgDocDatabase.GetMsgBySeqIndexIn1Doc(ctx, userID, docID, seqs)
for _, msg := range msgs {
if msg.IsRead {
@@ -554,25 +391,8 @@ func (db *commonMsgDatabase) findMsgInfoBySeq(
return msgs, err
}
func (db *commonMsgDatabase) getMsgBySeqsRange(
ctx context.Context,
userID string,
conversationID string,
allSeqs []int64,
begin, end int64,
) (seqMsgs []*sdkws.MsgData, err error) {
log.ZDebug(
ctx,
"getMsgBySeqsRange",
"conversationID",
conversationID,
"allSeqs",
allSeqs,
"begin",
begin,
"end",
end,
)
func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID string, conversationID string, allSeqs []int64, begin, end int64) (seqMsgs []*sdkws.MsgData, err error) {
log.ZDebug(ctx, "getMsgBySeqsRange", "conversationID", conversationID, "allSeqs", allSeqs, "begin", begin, "end", end)
for docID, seqs := range db.msg.GetDocIDSeqsMap(conversationID, allSeqs) {
log.ZDebug(ctx, "getMsgBySeqsRange", "docID", docID, "seqs", seqs)
msgs, err := db.findMsgInfoBySeq(ctx, userID, docID, seqs)
@@ -589,12 +409,7 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(
return seqMsgs, nil
}
func (db *commonMsgDatabase) GetMsgBySeqsRange(
ctx context.Context,
userID string,
conversationID string,
begin, end, num, userMaxSeq int64,
) (int64, int64, []*sdkws.MsgData, error) {
func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) {
userMinSeq, err := db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
@@ -614,18 +429,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
}
log.ZDebug(
ctx,
"GetMsgBySeqsRange",
"userMinSeq",
userMinSeq,
"conMinSeq",
minSeq,
"conMaxSeq",
maxSeq,
"userMaxSeq",
userMaxSeq,
)
log.ZDebug(ctx, "GetMsgBySeqsRange", "userMinSeq", userMinSeq, "conMinSeq", minSeq, "conMaxSeq", maxSeq, "userMaxSeq", userMaxSeq)
if userMaxSeq != 0 {
if userMaxSeq < maxSeq {
maxSeq = userMaxSeq
@@ -675,18 +479,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(
cacheDelNum += 1
}
}
log.ZDebug(
ctx,
"get delSeqs from redis",
"delSeqs",
delSeqs,
"userID",
userID,
"conversationID",
conversationID,
"cacheDelNum",
cacheDelNum,
)
log.ZDebug(ctx, "get delSeqs from redis", "delSeqs", delSeqs, "userID", userID, "conversationID", conversationID, "cacheDelNum", cacheDelNum)
var reGetSeqsCache []int64
for i := 1; i <= cacheDelNum; {
newSeq := newBegin - int64(i)
@@ -706,15 +499,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(
if err != nil {
if err != redis.Nil {
prome.Add(prome.MsgPullFromRedisFailedCounter, len(failedSeqs2))
log.ZError(
ctx,
"get message from redis exception",
err,
"conversationID",
conversationID,
"seqs",
reGetSeqsCache,
)
log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", reGetSeqsCache)
}
}
failedSeqs = append(failedSeqs, failedSeqs2...)
@@ -740,12 +525,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(
return minSeq, maxSeq, successMsgs, nil
}
func (db *commonMsgDatabase) GetMsgBySeqs(
ctx context.Context,
userID string,
conversationID string,
seqs []int64,
) (int64, int64, []*sdkws.MsgData, error) {
func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (int64, int64, []*sdkws.MsgData, error) {
userMinSeq, err := db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
if err != nil && errs.Unwrap(err) != redis.Nil {
return 0, 0, nil, err
@@ -771,33 +551,10 @@ func (db *commonMsgDatabase) GetMsgBySeqs(
if err != nil {
if err != redis.Nil {
prome.Add(prome.MsgPullFromRedisFailedCounter, len(failedSeqs))
log.ZError(
ctx,
"get message from redis exception",
err,
"failedSeqs",
failedSeqs,
"conversationID",
conversationID,
)
log.ZError(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID)
}
}
log.ZInfo(
ctx,
"db.cache.GetMessagesBySeq",
"userID",
userID,
"conversationID",
conversationID,
"seqs",
seqs,
"successMsgs",
len(successMsgs),
"failedSeqs",
failedSeqs,
"conversationID",
conversationID,
)
log.ZInfo(ctx, "db.cache.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs", seqs, "successMsgs", len(successMsgs), "failedSeqs", failedSeqs, "conversationID", conversationID)
prome.Add(prome.MsgPullFromRedisSuccessCounter, len(successMsgs))
if len(failedSeqs) > 0 {
mongoMsgs, err := db.getMsgBySeqs(ctx, userID, conversationID, failedSeqs)
@@ -811,11 +568,7 @@ func (db *commonMsgDatabase) GetMsgBySeqs(
return minSeq, maxSeq, successMsgs, nil
}
func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(
ctx context.Context,
conversationID string,
remainTime int64,
) error {
func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error {
var delStruct delMsgRecursionStruct
var skip int64
minSeq, err := db.deleteMsgRecursion(ctx, conversationID, skip, &delStruct, remainTime)
@@ -835,6 +588,49 @@ func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(
return db.cache.SetMinSeq(ctx, conversationID, minSeq)
}
func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error) {
var index int64
for {
// from oldest 2 newest
msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1)
if err != nil || msgDocModel.DocID == "" {
if err != nil {
if err == unrelation.ErrMsgListNotExist {
log.ZDebug(ctx, "deleteMsgRecursion finished", "conversationID", conversationID, "userID", userID, "index", index)
} else {
log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index)
}
}
// 获取报错，或者获取不到了，物理删除并且返回seq delMongoMsgsPhysical(delStruct.delDocIDList), 结束递归
break
}
index++
//&& msgDocModel.Msg[0].Msg.SendTime > lastMsgDestructTime.UnixMilli()
if len(msgDocModel.Msg) > 0 {
for _, msg := range msgDocModel.Msg {
if msg != nil && msg.Msg != nil && msg.Msg.SendTime+destructTime*1000 <= time.Now().UnixMilli() {
if msg.Msg.SendTime > lastMsgDestructTime.UnixMilli() && !utils.Contain(userID, msg.DelList...) {
seqs = append(seqs, msg.Msg.Seq)
}
} else {
log.ZDebug(ctx, "deleteMsgRecursion finished", "conversationID", conversationID, "userID", userID, "index", index)
break
}
}
}
}
log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversationID, "userID", userID, "seqs", seqs)
if len(seqs) > 0 {
latestSeq := seqs[len(seqs)-1]
if err := db.cache.SetConversationUserMinSeq(ctx, conversationID, userID, latestSeq); err != nil {
return nil, err
}
}
return seqs, nil
}
// this is struct for recursion
type delMsgRecursionStruct struct {
minSeq int64
@@ -849,26 +645,13 @@ func (d *delMsgRecursionStruct) getSetMinSeq() int64 {
// seq 70
// set minSeq 21
// recursion 删除list并且返回设置的最小seq
func (db *commonMsgDatabase) deleteMsgRecursion(
ctx context.Context,
conversationID string,
index int64,
delStruct *delMsgRecursionStruct,
remainTime int64,
) (int64, error) {
func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversationID string, index int64, delStruct *delMsgRecursionStruct, remainTime int64) (int64, error) {
// find from oldest list
msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1)
if err != nil || msgDocModel.DocID == "" {
if err != nil {
if err == unrelation.ErrMsgListNotExist {
log.ZDebug(
ctx,
"deleteMsgRecursion ErrMsgListNotExist",
"conversationID",
conversationID,
"index:",
index,
)
log.ZDebug(ctx, "deleteMsgRecursion ErrMsgListNotExist", "conversationID", conversationID, "index:", index)
} else {
log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index)
}
@@ -880,23 +663,11 @@ func (db *commonMsgDatabase) deleteMsgRecursion(
}
return delStruct.getSetMinSeq() + 1, nil
}
log.ZDebug(
ctx,
"doc info",
"conversationID",
conversationID,
"index",
index,
"docID",
msgDocModel.DocID,
"len",
len(msgDocModel.Msg),
)
log.ZDebug(ctx, "doc info", "conversationID", conversationID, "index", index, "docID", msgDocModel.DocID, "len", len(msgDocModel.Msg))
if int64(len(msgDocModel.Msg)) > db.msg.GetSingleGocMsgNum() {
log.ZWarn(ctx, "msgs too large", nil, "lenth", len(msgDocModel.Msg), "docID:", msgDocModel.DocID)
}
if msgDocModel.IsFull() &&
msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < utils.GetCurrentTimestampByMill() {
if msgDocModel.IsFull() && msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < utils.GetCurrentTimestampByMill() {
log.ZDebug(ctx, "doc is full and all msg is expired", "docID", msgDocModel.DocID)
delStruct.delDocIDs = append(delStruct.delDocIDs, msgDocModel.DocID)
delStruct.minSeq = msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.Seq
@@ -933,11 +704,7 @@ func (db *commonMsgDatabase) deleteMsgRecursion(
return seq, err
}
func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(
ctx context.Context,
conversationID string,
allSeqs []int64,
) error {
func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, allSeqs []int64) error {
if err := db.cache.DeleteMessages(ctx, conversationID, allSeqs); err != nil {
return err
}
@@ -953,12 +720,7 @@ func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(
return nil
}
func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(
ctx context.Context,
userID string,
conversationID string,
seqs []int64,
) error {
func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error {
cachedMsgs, _, err := db.cache.GetMessagesBySeq(ctx, conversationID, seqs)
if err != nil && errs.Unwrap(err) != redis.Nil {
log.ZWarn(ctx, "DeleteUserMsgsBySeqs", err, "conversationID", conversationID, "seqs", seqs)
@@ -1027,70 +789,31 @@ func (db *commonMsgDatabase) GetMinSeqs(ctx context.Context, conversationIDs []s
func (db *commonMsgDatabase) GetMinSeq(ctx context.Context, conversationID string) (int64, error) {
return db.cache.GetMinSeq(ctx, conversationID)
}
func (db *commonMsgDatabase) GetConversationUserMinSeq(
ctx context.Context,
conversationID string,
userID string,
) (int64, error) {
func (db *commonMsgDatabase) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
return db.cache.GetConversationUserMinSeq(ctx, conversationID, userID)
}
func (db *commonMsgDatabase) GetConversationUserMinSeqs(
ctx context.Context,
conversationID string,
userIDs []string,
) (map[string]int64, error) {
func (db *commonMsgDatabase) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) {
return db.cache.GetConversationUserMinSeqs(ctx, conversationID, userIDs)
}
func (db *commonMsgDatabase) SetConversationUserMinSeq(
ctx context.Context,
conversationID string,
userID string,
minSeq int64,
) error {
func (db *commonMsgDatabase) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error {
return db.cache.SetConversationUserMinSeq(ctx, conversationID, userID, minSeq)
}
func (db *commonMsgDatabase) SetConversationUserMinSeqs(
ctx context.Context,
conversationID string,
seqs map[string]int64,
) (err error) {
func (db *commonMsgDatabase) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) {
return db.cache.SetConversationUserMinSeqs(ctx, conversationID, seqs)
}
func (db *commonMsgDatabase) SetUserConversationsMinSeqs(
ctx context.Context,
userID string,
seqs map[string]int64,
) error {
func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error {
return db.cache.SetUserConversationsMinSeqs(ctx, userID, seqs)
}
func (db *commonMsgDatabase) UserSetHasReadSeqs(
ctx context.Context,
userID string,
hasReadSeqs map[string]int64,
) error {
func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error {
return db.cache.UserSetHasReadSeqs(ctx, userID, hasReadSeqs)
}
func (db *commonMsgDatabase) SetHasReadSeq(
ctx context.Context,
userID string,
conversationID string,
hasReadSeq int64,
) error {
func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error {
return db.cache.SetHasReadSeq(ctx, userID, conversationID, hasReadSeq)
}
func (db *commonMsgDatabase) GetHasReadSeqs(
ctx context.Context,
userID string,
conversationIDs []string,
) (map[string]int64, error) {
func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) {
return db.cache.GetHasReadSeqs(ctx, userID, conversationIDs)
}
@@ -1106,10 +829,7 @@ func (db *commonMsgDatabase) GetSendMsgStatus(ctx context.Context, id string) (i
return db.cache.GetSendMsgStatus(ctx, id)
}
func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(
ctx context.Context,
conversationID string,
) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) {
func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) {
minSeqMongo, maxSeqMongo, err = db.GetMinMaxSeqMongo(ctx, conversationID)
if err != nil {
return
@@ -1125,17 +845,11 @@ func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(
return
}
func (db *commonMsgDatabase) GetMongoMaxAndMinSeq(
ctx context.Context,
conversationID string,
) (maxSeq, minSeq int64, err error) {
func (db *commonMsgDatabase) GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) {
return db.GetMinMaxSeqMongo(ctx, conversationID)
}
func (db *commonMsgDatabase) GetMinMaxSeqMongo(
ctx context.Context,
conversationID string,
) (minSeqMongo, maxSeqMongo int64, err error) {
func (db *commonMsgDatabase) GetMinMaxSeqMongo(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) {
oldestMsgMongo, err := db.msgDocDatabase.GetOldestMsg(ctx, conversationID)
if err != nil {
return
@@ -1149,124 +863,10 @@ func (db *commonMsgDatabase) GetMinMaxSeqMongo(
return
}
func (db *commonMsgDatabase) JudgeMessageReactionExist(
ctx context.Context,
clientMsgID string,
sessionType int32,
) (bool, error) {
return db.cache.JudgeMessageReactionExist(ctx, clientMsgID, sessionType)
func (db *commonMsgDatabase) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*unRelationTb.UserCount, dateCount map[string]int64, err error) {
return db.msgDocDatabase.RangeUserSendCount(ctx, start, end, group, ase, pageNumber, showNumber)
}
func (db *commonMsgDatabase) SetMessageTypeKeyValue(
ctx context.Context,
clientMsgID string,
sessionType int32,
typeKey, value string,
) error {
return db.cache.SetMessageTypeKeyValue(ctx, clientMsgID, sessionType, typeKey, value)
}
func (db *commonMsgDatabase) SetMessageReactionExpire(
ctx context.Context,
clientMsgID string,
sessionType int32,
expiration time.Duration,
) (bool, error) {
return db.cache.SetMessageReactionExpire(ctx, clientMsgID, sessionType, expiration)
}
func (db *commonMsgDatabase) GetMessageTypeKeyValue(
ctx context.Context,
clientMsgID string,
sessionType int32,
typeKey string,
) (string, error) {
return db.cache.GetMessageTypeKeyValue(ctx, clientMsgID, sessionType, typeKey)
}
func (db *commonMsgDatabase) GetOneMessageAllReactionList(
ctx context.Context,
clientMsgID string,
sessionType int32,
) (map[string]string, error) {
return db.cache.GetOneMessageAllReactionList(ctx, clientMsgID, sessionType)
}
func (db *commonMsgDatabase) DeleteOneMessageKey(
ctx context.Context,
clientMsgID string,
sessionType int32,
subKey string,
) error {
return db.cache.DeleteOneMessageKey(ctx, clientMsgID, sessionType, subKey)
}
func (db *commonMsgDatabase) InsertOrUpdateReactionExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
msgFirstModifyTime int64,
reactionExtensions map[string]*sdkws.KeyValue,
) error {
return db.extendMsgDatabase.InsertOrUpdateReactionExtendMsgSet(
ctx,
conversationID,
sessionType,
clientMsgID,
msgFirstModifyTime,
db.extendMsgSetModel.Pb2Model(reactionExtensions),
)
}
func (db *commonMsgDatabase) GetExtendMsg(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
maxMsgUpdateTime int64,
) (*pbMsg.ExtendMsg, error) {
extendMsgSet, err := db.extendMsgDatabase.GetExtendMsgSet(ctx, conversationID, sessionType, maxMsgUpdateTime)
if err != nil {
return nil, err
}
extendMsg, ok := extendMsgSet.ExtendMsgs[clientMsgID]
if !ok {
return nil, errs.ErrRecordNotFound.Wrap(fmt.Sprintf("cant find client msg id: %s", clientMsgID))
}
reactionExtensionList := make(map[string]*pbMsg.KeyValueResp)
for key, model := range extendMsg.ReactionExtensionList {
reactionExtensionList[key] = &pbMsg.KeyValueResp{
KeyValue: &sdkws.KeyValue{
TypeKey: model.TypeKey,
Value: model.Value,
LatestUpdateTime: model.LatestUpdateTime,
},
}
}
return &pbMsg.ExtendMsg{
ReactionExtensions: reactionExtensionList,
ClientMsgID: extendMsg.ClientMsgID,
MsgFirstModifyTime: extendMsg.MsgFirstModifyTime,
AttachedInfo: extendMsg.AttachedInfo,
Ex: extendMsg.Ex,
}, nil
}
func (db *commonMsgDatabase) DeleteReactionExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
msgFirstModifyTime int64,
reactionExtensions map[string]*sdkws.KeyValue,
) error {
return db.extendMsgDatabase.DeleteReactionExtendMsgSet(
ctx,
conversationID,
sessionType,
clientMsgID,
msgFirstModifyTime,
db.extendMsgSetModel.Pb2Model(reactionExtensions),
)
func (db *commonMsgDatabase) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*unRelationTb.GroupCount, dateCount map[string]int64, err error) {
return db.msgDocDatabase.RangeGroupSendCount(ctx, start, end, ase, pageNumber, showNumber)
}
pkg/common/db/controller/s3.go
+75
View File
@@ -0,0 +1,75 @@
package controller
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cont"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"path/filepath"
"time"
)
type S3Database interface {
PartLimit() *s3.PartLimit
PartSize(ctx context.Context, size int64) (int64, error)
AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error)
InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error)
CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error)
AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error)
SetObject(ctx context.Context, info *relation.ObjectModel) error
}
func NewS3Database(s3 s3.Interface, obj relation.ObjectInfoModelInterface) S3Database {
return &s3Database{
s3: cont.New(s3),
obj: obj,
}
}
type s3Database struct {
s3 *cont.Controller
obj relation.ObjectInfoModelInterface
}
func (s *s3Database) PartSize(ctx context.Context, size int64) (int64, error) {
return s.s3.PartSize(ctx, size)
}
func (s *s3Database) PartLimit() *s3.PartLimit {
return s.s3.PartLimit()
}
func (s *s3Database) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) {
return s.s3.AuthSign(ctx, uploadID, partNumbers)
}
func (s *s3Database) InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error) {
return s.s3.InitiateUpload(ctx, hash, size, expire, maxParts)
}
func (s *s3Database) CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error) {
return s.s3.CompleteUpload(ctx, uploadID, parts)
}
func (s *s3Database) SetObject(ctx context.Context, info *relation.ObjectModel) error {
return s.obj.SetObject(ctx, info)
}
func (s *s3Database) AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error) {
obj, err := s.obj.Take(ctx, name)
if err != nil {
return time.Time{}, "", err
}
opt := &s3.AccessURLOption{
ContentType: obj.ContentType,
}
if filename := filepath.Base(obj.Name); filename != "" {
opt.ContentDisposition = `attachment; filename=` + filename
}
expireTime := time.Now().Add(expire)
rawURL, err := s.s3.AccessURL(ctx, obj.Key, expire, opt)
if err != nil {
return time.Time{}, "", err
}
return expireTime, rawURL, nil
}
pkg/common/db/controller/storage.go
-567
View File
@@ -1,567 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controller
import (
"bytes"
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
"path"
"strconv"
"time"
"github.com/google/uuid"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/obj"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
)
const (
hashPrefix = "hash"
tempPrefix = "temp"
fragmentPrefix = "fragment_"
urlsName = "urls.json"
)
type S3Database interface {
ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error)
GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error)
ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (*third.ConfirmPutResp, error)
GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error)
GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error)
CleanExpirationObject(ctx context.Context, t time.Time)
}
func NewS3Database(
obj obj.Interface,
hash relation.ObjectHashModelInterface,
info relation.ObjectInfoModelInterface,
put relation.ObjectPutModelInterface,
url *url.URL,
) S3Database {
return &s3Database{
url: url,
obj: obj,
hash: hash,
info: info,
put: put,
}
}
type s3Database struct {
url *url.URL
obj obj.Interface
hash relation.ObjectHashModelInterface
info relation.ObjectInfoModelInterface
put relation.ObjectPutModelInterface
}
// today 今天的日期
func (c *s3Database) today() string {
return time.Now().Format("20060102")
}
// fragmentName 根据序号生成文件名
func (c *s3Database) fragmentName(index int) string {
return fragmentPrefix + strconv.Itoa(index+1)
}
// getFragmentNum 获取分片大小和分片数量
func (c *s3Database) getFragmentNum(fragmentSize int64, objectSize int64) (int64, int) {
if size := c.obj.MinFragmentSize(); fragmentSize < size {
fragmentSize = size
}
if fragmentSize <= 0 || objectSize <= fragmentSize {
return objectSize, 1
} else {
num := int(objectSize / fragmentSize)
if objectSize%fragmentSize > 0 {
num++
}
if n := c.obj.MaxFragmentNum(); num > n {
num = n
}
return fragmentSize, num
}
}
func (c *s3Database) CheckHash(hash string) error {
val, err := hex.DecodeString(hash)
if err != nil {
return err
}
if len(val) != md5.Size {
return errs.ErrArgs.Wrap("invalid hash")
}
return nil
}
func (c *s3Database) urlName(name string) string {
u := url.URL{
Scheme: c.url.Scheme,
Opaque: c.url.Opaque,
User: c.url.User,
Host: c.url.Host,
Path: c.url.Path,
RawPath: c.url.RawPath,
ForceQuery: c.url.ForceQuery,
RawQuery: c.url.RawQuery,
Fragment: c.url.Fragment,
RawFragment: c.url.RawFragment,
}
v := make(url.Values, 1)
v.Set("name", name)
u.RawQuery = v.Encode()
return u.String()
}
func (c *s3Database) UUID() string {
return uuid.New().String()
}
func (c *s3Database) HashName(hash string) string {
return path.Join(hashPrefix, hash+"_"+c.today()+"_"+c.UUID())
}
func (c *s3Database) isNotFound(err error) bool {
return relation.IsNotFound(err)
}
func (c *s3Database) ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error) {
if err := c.CheckHash(req.Hash); err != nil {
return nil, err
}
if err := c.obj.CheckName(req.Name); err != nil {
return nil, err
}
if req.ValidTime != 0 && req.ValidTime <= time.Now().UnixMilli() {
return nil, errors.New("invalid ValidTime")
}
var expirationTime *time.Time
if req.ValidTime != 0 {
expirationTime = utils.ToPtr(time.UnixMilli(req.ValidTime))
}
if hash, err := c.hash.Take(ctx, req.Hash, c.obj.Name()); err == nil {
o := relation.ObjectInfoModel{
Name: req.Name,
Hash: hash.Hash,
ValidTime: expirationTime,
ContentType: req.ContentType,
CreateTime: time.Now(),
}
if err := c.info.SetObject(ctx, &o); err != nil {
return nil, err
}
return &third.ApplyPutResp{Url: c.urlName(o.Name)}, nil // 服务器已存在
} else if !c.isNotFound(err) {
return nil, err
}
// 新上传
var fragmentNum int
const effective = time.Hour * 24 * 2
req.FragmentSize, fragmentNum = c.getFragmentNum(req.FragmentSize, req.Size)
put := relation.ObjectPutModel{
PutID: req.PutID,
Hash: req.Hash,
Name: req.Name,
ObjectSize: req.Size,
ContentType: req.ContentType,
FragmentSize: req.FragmentSize,
ValidTime: expirationTime,
EffectiveTime: time.Now().Add(effective),
}
if put.PutID == "" {
put.PutID = c.UUID()
}
if v, err := c.put.Take(ctx, put.PutID); err == nil {
now := time.Now().UnixMilli()
if v.EffectiveTime.UnixMilli() <= now {
if err := c.put.DelPut(ctx, []string{v.PutID}); err != nil {
return nil, err
}
} else {
return nil, errs.ErrDuplicateKey.Wrap(fmt.Sprintf("duplicate put id %s", put.PutID))
}
} else if !c.isNotFound(err) {
return nil, err
}
put.Path = path.Join(tempPrefix, c.today(), req.Hash, put.PutID)
putURLs := make([]string, 0, fragmentNum)
for i := 0; i < fragmentNum; i++ {
url, err := c.obj.PresignedPutURL(ctx, &obj.ApplyPutArgs{
Bucket: c.obj.TempBucket(),
Name: path.Join(put.Path, c.fragmentName(i)),
Effective: effective,
MaxObjectSize: req.FragmentSize,
})
if err != nil {
return nil, err
}
putURLs = append(putURLs, url)
}
urlsJsonData, err := json.Marshal(putURLs)
if err != nil {
return nil, err
}
t := md5.Sum(urlsJsonData)
put.PutURLsHash = hex.EncodeToString(t[:])
_, err = c.obj.PutObject(
ctx,
&obj.BucketObject{Bucket: c.obj.TempBucket(), Name: path.Join(put.Path, urlsName)},
bytes.NewReader(urlsJsonData),
int64(len(urlsJsonData)),
)
if err != nil {
return nil, err
}
put.CreateTime = time.Now()
if err := c.put.Create(ctx, []*relation.ObjectPutModel{&put}); err != nil {
return nil, err
}
return &third.ApplyPutResp{
PutID: put.PutID,
FragmentSize: put.FragmentSize,
PutURLs: putURLs,
ValidTime: put.EffectiveTime.UnixMilli(),
}, nil
}
func (c *s3Database) GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error) {
up, err := c.put.Take(ctx, req.PutID)
if err != nil {
return nil, err
}
reader, err := c.obj.GetObject(
ctx,
&obj.BucketObject{Bucket: c.obj.TempBucket(), Name: path.Join(up.Path, urlsName)},
)
if err != nil {
return nil, err
}
urlsData, err := io.ReadAll(reader)
if err != nil {
return nil, err
}
t := md5.Sum(urlsData)
if h := hex.EncodeToString(t[:]); h != up.PutURLsHash {
return nil, fmt.Errorf("invalid put urls hash %s %s", h, up.PutURLsHash)
}
var urls []string
if err := json.Unmarshal(urlsData, &urls); err != nil {
return nil, err
}
_, fragmentNum := c.getFragmentNum(up.FragmentSize, up.ObjectSize)
if len(urls) != fragmentNum {
return nil, fmt.Errorf("invalid urls length %d fragment %d", len(urls), fragmentNum)
}
fragments := make([]*third.GetPutFragment, fragmentNum)
for i := 0; i < fragmentNum; i++ {
name := path.Join(up.Path, c.fragmentName(i))
o, err := c.obj.GetObjectInfo(ctx, &obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: name,
})
if err != nil {
if c.obj.IsNotFound(err) {
fragments[i] = &third.GetPutFragment{Url: urls[i]}
continue
}
return nil, err
}
fragments[i] = &third.GetPutFragment{Size: o.Size, Hash: o.Hash, Url: urls[i]}
}
var validTime int64
if up.ValidTime != nil {
validTime = up.ValidTime.UnixMilli()
}
return &third.GetPutResp{
FragmentSize: up.FragmentSize,
Size: up.ObjectSize,
Name: up.Name,
Hash: up.Hash,
Fragments: fragments,
PutURLsHash: up.PutURLsHash,
ContentType: up.ContentType,
ValidTime: validTime,
}, nil
}
func (c *s3Database) ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (_ *third.ConfirmPutResp, _err error) {
put, err := c.put.Take(ctx, req.PutID)
if err != nil {
return nil, err
}
_, pack := c.getFragmentNum(put.FragmentSize, put.ObjectSize)
defer func() {
if _err == nil {
// 清理上传的碎片
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: put.Path})
if err != nil {
log.ZError(ctx, "deleteObject failed", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
}
}
}()
now := time.Now().UnixMilli()
if put.EffectiveTime.UnixMilli() < now {
return nil, errs.ErrFileUploadedExpired.Wrap("put expired")
}
if put.ValidTime != nil && put.ValidTime.UnixMilli() < now {
return nil, errs.ErrFileUploadedExpired.Wrap("object expired")
}
if hash, err := c.hash.Take(ctx, put.Hash, c.obj.Name()); err == nil {
o := relation.ObjectInfoModel{
Name: put.Name,
Hash: hash.Hash,
ValidTime: put.ValidTime,
ContentType: put.ContentType,
CreateTime: time.Now(),
}
if err := c.info.SetObject(ctx, &o); err != nil {
return nil, err
}
defer func() {
err := c.obj.DeleteObject(ctx, &obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: put.Path,
})
if err != nil {
log.ZError(ctx, "DeleteObject", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
}
}()
// 服务端已存在
return &third.ConfirmPutResp{
Url: c.urlName(o.Name),
}, nil
} else if !c.isNotFound(err) {
return nil, err
}
src := make([]obj.BucketObject, pack)
for i := 0; i < pack; i++ {
name := path.Join(put.Path, c.fragmentName(i))
o, err := c.obj.GetObjectInfo(ctx, &obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: name,
})
if err != nil {
return nil, err
}
if i+1 == pack { // 最后一个
size := put.ObjectSize - put.FragmentSize*int64(i)
if size != o.Size {
return nil, fmt.Errorf("last fragment %d size %d not equal to %d hash %s", i, o.Size, size, o.Hash)
}
} else {
if o.Size != put.FragmentSize {
return nil, fmt.Errorf("fragment %d size %d not equal to %d hash %s", i, o.Size, put.FragmentSize, o.Hash)
}
}
src[i] = obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: name,
}
}
dst := &obj.BucketObject{
Bucket: c.obj.DataBucket(),
Name: c.HashName(put.Hash),
}
if len(src) == 1 { // 未分片直接触发copy
// 检查数据完整性,避免脏数据
o, err := c.obj.GetObjectInfo(ctx, &src[0])
if err != nil {
return nil, err
}
if put.ObjectSize != o.Size {
return nil, fmt.Errorf("size mismatching should %d reality %d", put.ObjectSize, o.Size)
}
if put.Hash != o.Hash {
return nil, fmt.Errorf("hash mismatching should %s reality %s", put.Hash, o.Hash)
}
if err := c.obj.CopyObject(ctx, &src[0], dst); err != nil {
return nil, err
}
} else {
tempBucket := &obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: path.Join(put.Path, "merge_"+c.UUID()),
}
defer func() { // 清理合成的文件
if err := c.obj.DeleteObject(ctx, tempBucket); err != nil {
log.ZError(ctx, "DeleteObject", err, "Bucket", tempBucket.Bucket, "Path", tempBucket.Name)
}
}()
err := c.obj.ComposeObject(ctx, src, tempBucket)
if err != nil {
return nil, err
}
info, err := c.obj.GetObjectInfo(ctx, tempBucket)
if err != nil {
return nil, err
}
if put.ObjectSize != info.Size {
return nil, fmt.Errorf("size mismatch should %d reality %d", put.ObjectSize, info.Size)
}
if put.Hash != info.Hash {
return nil, fmt.Errorf("hash mismatch should %s reality %s", put.Hash, info.Hash)
}
if err := c.obj.CopyObject(ctx, tempBucket, dst); err != nil {
return nil, err
}
}
h := &relation.ObjectHashModel{
Hash: put.Hash,
Engine: c.obj.Name(),
Size: put.ObjectSize,
Bucket: c.obj.DataBucket(),
Name: dst.Name,
CreateTime: time.Now(),
}
if err := c.hash.Create(ctx, []*relation.ObjectHashModel{h}); err != nil {
return nil, err
}
o := &relation.ObjectInfoModel{
Name: put.Name,
Hash: put.Hash,
ContentType: put.ContentType,
ValidTime: put.ValidTime,
CreateTime: time.Now(),
}
if err := c.info.SetObject(ctx, o); err != nil {
return nil, err
}
if err := c.put.DelPut(ctx, []string{put.PutID}); err != nil {
log.ZError(ctx, "DelPut", err, "PutID", put.PutID)
}
return &third.ConfirmPutResp{
Url: c.urlName(o.Name),
}, nil
}
func (c *s3Database) GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error) {
info, err := c.info.Take(ctx, req.Name)
if err != nil {
return nil, err
}
if info.ValidTime != nil && info.ValidTime.Before(time.Now()) {
return nil, errs.ErrRecordNotFound.Wrap("object expired")
}
hash, err := c.hash.Take(ctx, info.Hash, c.obj.Name())
if err != nil {
return nil, err
}
opt := obj.HeaderOption{ContentType: info.ContentType}
if req.Attachment {
opt.Filename = info.Name
}
u, err := c.obj.PresignedGetURL(ctx, hash.Bucket, hash.Name, time.Duration(req.Expires)*time.Millisecond, &opt)
if err != nil {
return nil, err
}
return &third.GetUrlResp{
Url: u,
Size: hash.Size,
Hash: hash.Hash,
}, nil
}
func (c *s3Database) CleanExpirationObject(ctx context.Context, t time.Time) {
// 清理上传产生的临时文件
c.cleanPutTemp(ctx, t, 10)
// 清理hash引用全过期的文件
c.cleanExpirationObject(ctx, t)
// 清理没有引用的hash对象
c.clearNoCitation(ctx, c.obj.Name(), 10)
}
func (c *s3Database) cleanPutTemp(ctx context.Context, t time.Time, num int) {
for {
puts, err := c.put.FindExpirationPut(ctx, t, num)
if err != nil {
log.ZError(ctx, "FindExpirationPut", err, "Time", t, "Num", num)
return
}
if len(puts) == 0 {
return
}
for _, put := range puts {
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: put.Path})
if err != nil {
log.ZError(ctx, "DeleteObject", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
return
}
}
ids := utils.Slice(puts, func(e *relation.ObjectPutModel) string { return e.PutID })
err = c.put.DelPut(ctx, ids)
if err != nil {
log.ZError(ctx, "DelPut", err, "PutID", ids)
return
}
}
}
func (c *s3Database) cleanExpirationObject(ctx context.Context, t time.Time) {
err := c.info.DeleteExpiration(ctx, t)
if err != nil {
log.ZError(ctx, "DeleteExpiration", err, "Time", t)
}
}
func (c *s3Database) clearNoCitation(ctx context.Context, engine string, limit int) {
for {
list, err := c.hash.DeleteNoCitation(ctx, engine, limit)
if err != nil {
log.ZError(ctx, "DeleteNoCitation", err, "Engine", engine, "Limit", limit)
return
}
if len(list) == 0 {
return
}
var hasErr bool
for _, h := range list {
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: h.Bucket, Name: h.Name})
if err != nil {
hasErr = true
log.ZError(ctx, "DeleteObject", err, "Bucket", h.Bucket, "Path", h.Name)
continue
}
}
if hasErr {
return
}
}
}
func (c *s3Database) GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error) {
if err := c.CheckHash(req.Hash); err != nil {
return nil, err
}
o, err := c.hash.Take(ctx, req.Hash, c.obj.Name())
if err != nil {
return nil, err
}
return &third.GetHashInfoResp{
Hash: o.Hash,
Size: o.Size,
}, nil
}
pkg/common/db/controller/user.go
+3 -3
View File
@@ -45,7 +45,7 @@ type UserDatabase interface {
//函数内部先查询db中是否存在，存在则什么都不做；不存在则插入
InitOnce(ctx context.Context, users []*relation.UserModel) (err error)
// 获取用户总数
CountTotal(ctx context.Context) (int64, error)
CountTotal(ctx context.Context, before *time.Time) (int64, error)
// 获取范围内用户增量
CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
}
@@ -151,8 +151,8 @@ func (u *userDatabase) GetAllUserID(ctx context.Context) (userIDs []string, err
return u.userDB.GetAllUserID(ctx)
}
func (u *userDatabase) CountTotal(ctx context.Context) (count int64, err error) {
return u.userDB.CountTotal(ctx)
func (u *userDatabase) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) {
return u.userDB.CountTotal(ctx, before)
}
func (u *userDatabase) CountRangeEverydayTotal(
pkg/common/db/obj/minio.go
-258
View File
@@ -1,258 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package obj
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/s3utils"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
)
func NewMinioInterface() (Interface, error) {
conf := config.Config.Object.Minio
u, err := url.Parse(conf.Endpoint)
if err != nil {
return nil, fmt.Errorf("minio endpoint parse %w", err)
}
if u.Scheme != "http" && u.Scheme != "https" {
return nil, fmt.Errorf("invalid minio endpoint scheme %s", u.Scheme)
}
client, err := minio.New(u.Host, &minio.Options{
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, ""),
Secure: u.Scheme == "https",
})
if err != nil {
return nil, fmt.Errorf("minio new client %w", err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*20)
defer cancel()
initBucket := func(ctx context.Context) error {
for _, bucket := range utils.Distinct([]string{conf.TempBucket, conf.DataBucket}) {
exists, err := client.BucketExists(ctx, bucket)
if err != nil {
return fmt.Errorf("minio bucket %s exists %w", bucket, err)
}
if exists {
continue
}
opt := minio.MakeBucketOptions{
Region: conf.Location,
ObjectLocking: conf.IsDistributedMod,
}
if err := client.MakeBucket(ctx, bucket, opt); err != nil {
return fmt.Errorf("minio make bucket %s %w", bucket, err)
}
}
return nil
}
if err := initBucket(ctx); err != nil {
fmt.Println("minio init error:", err)
}
return &minioImpl{
client: client,
tempBucket: conf.TempBucket,
dataBucket: conf.DataBucket,
}, nil
}
type minioImpl struct {
tempBucket string // 上传桶
dataBucket string // 永久桶
urlstr string // 访问地址
client *minio.Client
}
func (m *minioImpl) Name() string {
return "minio"
}
func (m *minioImpl) MinFragmentSize() int64 {
return 1024 * 1024 * 5 // 每个分片最小大小 minio.absMinPartSize
}
func (m *minioImpl) MaxFragmentNum() int {
return 1000 // 最大分片数量 minio.maxPartsCount
}
func (m *minioImpl) MinExpirationTime() time.Duration {
return time.Hour * 24
}
func (m *minioImpl) TempBucket() string {
return m.tempBucket
}
func (m *minioImpl) DataBucket() string {
return m.dataBucket
}
func (m *minioImpl) PresignedGetURL(
ctx context.Context,
bucket string,
name string,
expires time.Duration,
opt *HeaderOption,
) (string, error) {
var reqParams url.Values
if opt != nil {
reqParams = make(url.Values)
if opt.ContentType != "" {
reqParams.Set("response-content-type", opt.ContentType)
}
if opt.Filename != "" {
reqParams.Set("response-content-disposition", "attachment;filename="+opt.Filename)
}
}
u, err := m.client.PresignedGetObject(ctx, bucket, name, expires, reqParams)
if err != nil {
return "", err
}
return u.String(), nil
}
func (m *minioImpl) PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error) {
if args.Effective <= 0 {
return "", errors.New("EffectiveTime <= 0")
}
_, err := m.GetObjectInfo(ctx, &BucketObject{
Bucket: m.tempBucket,
Name: args.Name,
})
if err == nil {
return "", fmt.Errorf("minio bucket %s name %s already exists", args.Bucket, args.Name)
} else if !m.IsNotFound(err) {
return "", err
}
u, err := m.client.PresignedPutObject(ctx, m.tempBucket, args.Name, args.Effective)
if err != nil {
return "", fmt.Errorf("minio apply error: %w", err)
}
return u.String(), nil
}
func (m *minioImpl) GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error) {
info, err := m.client.StatObject(ctx, args.Bucket, args.Name, minio.StatObjectOptions{})
if err != nil {
return nil, err
}
return &ObjectInfo{
Size: info.Size,
Hash: info.ETag,
}, nil
}
func (m *minioImpl) CopyObject(ctx context.Context, src *BucketObject, dst *BucketObject) error {
_, err := m.client.CopyObject(ctx, minio.CopyDestOptions{
Bucket: dst.Bucket,
Object: dst.Name,
}, minio.CopySrcOptions{
Bucket: src.Bucket,
Object: src.Name,
})
return err
}
func (m *minioImpl) DeleteObject(ctx context.Context, info *BucketObject) error {
return m.client.RemoveObject(ctx, info.Bucket, info.Name, minio.RemoveObjectOptions{})
}
func (m *minioImpl) MoveObjectInfo(ctx context.Context, src *BucketObject, dst *BucketObject) error {
if err := m.CopyObject(ctx, src, dst); err != nil {
return err
}
return m.DeleteObject(ctx, src)
}
func (m *minioImpl) ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error {
destOptions := minio.CopyDestOptions{
Bucket: dst.Bucket,
Object: dst.Name + ".temp",
}
sources := make([]minio.CopySrcOptions, len(src))
for i, s := range src {
sources[i] = minio.CopySrcOptions{
Bucket: s.Bucket,
Object: s.Name,
}
}
_, err := m.client.ComposeObject(ctx, destOptions, sources...)
if err != nil {
return err
}
return m.MoveObjectInfo(ctx, &BucketObject{
Bucket: destOptions.Bucket,
Name: destOptions.Object,
}, &BucketObject{
Bucket: dst.Bucket,
Name: dst.Name,
})
}
func (m *minioImpl) IsNotFound(err error) bool {
if err == nil {
return false
}
switch e := err.(type) {
case minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
case *minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (m *minioImpl) PutObject(
ctx context.Context,
info *BucketObject,
reader io.Reader,
size int64,
) (*ObjectInfo, error) {
update, err := m.client.PutObject(ctx, info.Bucket, info.Name, reader, size, minio.PutObjectOptions{})
if err != nil {
return nil, err
}
return &ObjectInfo{
Size: update.Size,
Hash: update.ETag,
}, nil
}
func (m *minioImpl) GetObject(ctx context.Context, info *BucketObject) (SizeReader, error) {
object, err := m.client.GetObject(ctx, info.Bucket, info.Name, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
stat, err := object.Stat()
if err != nil {
return nil, err
}
return NewSizeReader(object, stat.Size), nil
}
func (m *minioImpl) CheckName(name string) error {
return s3utils.CheckValidObjectName(name)
}
pkg/common/db/obj/obj.go
-110
View File
@@ -1,110 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package obj
import (
"context"
"io"
"net/http"
"time"
)
type BucketObject struct {
Bucket string `json:"bucket"`
Name string `json:"name"`
}
type ApplyPutArgs struct {
Bucket string
Name string
Effective time.Duration // 申请有效时间
Header http.Header // header
MaxObjectSize int64
}
type HeaderOption struct {
ContentType string
Filename string
}
type ObjectInfo struct {
Size int64
Hash string
}
type SizeReader interface {
io.ReadCloser
Size() int64
}
func NewSizeReader(r io.ReadCloser, size int64) SizeReader {
if r == nil {
return nil
}
return &sizeReader{
size: size,
ReadCloser: r,
}
}
type sizeReader struct {
size int64
io.ReadCloser
}
func (r *sizeReader) Size() int64 {
return r.size
}
type Interface interface {
// Name 存储名字
Name() string
// MinFragmentSize 最小允许的分片大小
MinFragmentSize() int64
// MaxFragmentNum 最大允许的分片数量
MaxFragmentNum() int
// MinExpirationTime 最小过期时间
MinExpirationTime() time.Duration
// TempBucket 临时桶名，用于上传
TempBucket() string
// DataBucket 永久存储的桶名
DataBucket() string
// PresignedGetURL 通过桶名和对象名返回URL
PresignedGetURL(
ctx context.Context,
bucket string,
name string,
expires time.Duration,
opt *HeaderOption,
) (string, error)
// PresignedPutURL 申请上传,返回PUT的上传地址
PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error)
// GetObjectInfo 获取对象信息
GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error)
// CopyObject 复制对象
CopyObject(ctx context.Context, src *BucketObject, dst *BucketObject) error
// DeleteObject 删除对象(不存在返回nil)
DeleteObject(ctx context.Context, info *BucketObject) error
// ComposeObject 合并对象
ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error
// IsNotFound 判断是不是不存在导致的错误
IsNotFound(err error) bool
// CheckName 检查名字是否可用
CheckName(name string) error
// PutObject 上传文件
PutObject(ctx context.Context, info *BucketObject, reader io.Reader, size int64) (*ObjectInfo, error)
// GetObject 下载文件
GetObject(ctx context.Context, info *BucketObject) (SizeReader, error)
}
pkg/common/db/relation/conversation_model.go
+29 -151
View File
@@ -1,27 +1,12 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package relation
import (
"context"
"gorm.io/gorm"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"gorm.io/gorm"
)
type ConversationGorm struct {
@@ -44,173 +29,66 @@ func (c *ConversationGorm) Delete(ctx context.Context, groupIDs []string) (err e
return utils.Wrap(c.db(ctx).Where("group_id in (?)", groupIDs).Delete(&relation.ConversationModel{}).Error, "")
}
func (c *ConversationGorm) UpdateByMap(
ctx context.Context,
userIDList []string,
conversationID string,
args map[string]interface{},
) (rows int64, err error) {
func (c *ConversationGorm) UpdateByMap(ctx context.Context, userIDList []string, conversationID string, args map[string]interface{}) (rows int64, err error) {
result := c.db(ctx).Where("owner_user_id IN (?) and conversation_id=?", userIDList, conversationID).Updates(args)
return result.RowsAffected, utils.Wrap(result.Error, "")
}
func (c *ConversationGorm) Update(ctx context.Context, conversation *relation.ConversationModel) (err error) {
return utils.Wrap(
c.db(ctx).
Where("owner_user_id = ? and conversation_id = ?", conversation.OwnerUserID, conversation.ConversationID).
Updates(conversation).
Error,
"",
)
return utils.Wrap(c.db(ctx).Where("owner_user_id = ? and conversation_id = ?", conversation.OwnerUserID, conversation.ConversationID).Updates(conversation).Error, "")
}
func (c *ConversationGorm) Find(
ctx context.Context,
ownerUserID string,
conversationIDs []string,
) (conversations []*relation.ConversationModel, err error) {
err = utils.Wrap(
c.db(ctx).
Where("owner_user_id=? and conversation_id IN (?)", ownerUserID, conversationIDs).
Find(&conversations).
Error,
"",
)
func (c *ConversationGorm) Find(ctx context.Context, ownerUserID string, conversationIDs []string) (conversations []*relation.ConversationModel, err error) {
err = utils.Wrap(c.db(ctx).Where("owner_user_id=? and conversation_id IN (?)", ownerUserID, conversationIDs).Find(&conversations).Error, "")
return conversations, err
}
func (c *ConversationGorm) Take(
ctx context.Context,
userID, conversationID string,
) (conversation *relation.ConversationModel, err error) {
func (c *ConversationGorm) Take(ctx context.Context, userID, conversationID string) (conversation *relation.ConversationModel, err error) {
cc := &relation.ConversationModel{}
return cc, utils.Wrap(
c.db(ctx).Where("conversation_id = ? And owner_user_id = ?", conversationID, userID).Take(cc).Error,
"",
)
return cc, utils.Wrap(c.db(ctx).Where("conversation_id = ? And owner_user_id = ?", conversationID, userID).Take(cc).Error, "")
}
func (c *ConversationGorm) FindUserID(
ctx context.Context,
userIDs []string,
conversationIDs []string,
) (existUserID []string, err error) {
return existUserID, utils.Wrap(
c.db(ctx).
Where(" owner_user_id IN (?) and conversation_id in (?)", userIDs, conversationIDs).
Pluck("owner_user_id", &existUserID).
Error,
"",
)
func (c *ConversationGorm) FindUserID(ctx context.Context, userIDs []string, conversationIDs []string) (existUserID []string, err error) {
return existUserID, utils.Wrap(c.db(ctx).Where(" owner_user_id IN (?) and conversation_id in (?)", userIDs, conversationIDs).Pluck("owner_user_id", &existUserID).Error, "")
}
func (c *ConversationGorm) FindConversationID(
ctx context.Context,
userID string,
conversationIDList []string,
) (existConversationID []string, err error) {
return existConversationID, utils.Wrap(
c.db(ctx).
Where(" conversation_id IN (?) and owner_user_id=?", conversationIDList, userID).
Pluck("conversation_id", &existConversationID).
Error,
"",
)
func (c *ConversationGorm) FindConversationID(ctx context.Context, userID string, conversationIDList []string) (existConversationID []string, err error) {
return existConversationID, utils.Wrap(c.db(ctx).Where(" conversation_id IN (?) and owner_user_id=?", conversationIDList, userID).Pluck("conversation_id", &existConversationID).Error, "")
}
func (c *ConversationGorm) FindUserIDAllConversationID(
ctx context.Context,
userID string,
) (conversationIDList []string, err error) {
return conversationIDList, utils.Wrap(
c.db(ctx).Where("owner_user_id=?", userID).Pluck("conversation_id", &conversationIDList).Error,
"",
)
func (c *ConversationGorm) FindUserIDAllConversationID(ctx context.Context, userID string) (conversationIDList []string, err error) {
return conversationIDList, utils.Wrap(c.db(ctx).Where("owner_user_id=?", userID).Pluck("conversation_id", &conversationIDList).Error, "")
}
func (c *ConversationGorm) FindUserIDAllConversations(
ctx context.Context,
userID string,
) (conversations []*relation.ConversationModel, err error) {
func (c *ConversationGorm) FindUserIDAllConversations(ctx context.Context, userID string) (conversations []*relation.ConversationModel, err error) {
return conversations, utils.Wrap(c.db(ctx).Where("owner_user_id=?", userID).Find(&conversations).Error, "")
}
func (c *ConversationGorm) FindRecvMsgNotNotifyUserIDs(
ctx context.Context,
groupID string,
) (userIDs []string, err error) {
return userIDs, utils.Wrap(
c.db(ctx).
Where("group_id = ? and recv_msg_opt = ?", groupID, constant.ReceiveNotNotifyMessage).
Pluck("user_id", &userIDs).
Error,
"",
)
func (c *ConversationGorm) FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) (userIDs []string, err error) {
return userIDs, utils.Wrap(c.db(ctx).Where("group_id = ? and recv_msg_opt = ?", groupID, constant.ReceiveNotNotifyMessage).Pluck("user_id", &userIDs).Error, "")
}
func (c *ConversationGorm) FindSuperGroupRecvMsgNotNotifyUserIDs(
ctx context.Context,
groupID string,
) (userIDs []string, err error) {
return userIDs, utils.Wrap(
c.db(ctx).
Where("group_id = ? and recv_msg_opt = ? and conversation_type = ?", groupID, constant.ReceiveNotNotifyMessage, constant.SuperGroupChatType).
Pluck("user_id", &userIDs).
Error,
"",
)
func (c *ConversationGorm) FindSuperGroupRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) (userIDs []string, err error) {
return userIDs, utils.Wrap(c.db(ctx).Where("group_id = ? and recv_msg_opt = ? and conversation_type = ?", groupID, constant.ReceiveNotNotifyMessage, constant.SuperGroupChatType).Pluck("user_id", &userIDs).Error, "")
}
func (c *ConversationGorm) GetUserRecvMsgOpt(
ctx context.Context,
ownerUserID, conversationID string,
) (opt int, err error) {
func (c *ConversationGorm) GetUserRecvMsgOpt(ctx context.Context, ownerUserID, conversationID string) (opt int, err error) {
var conversation relation.ConversationModel
return int(
conversation.RecvMsgOpt,
), utils.Wrap(
c.db(ctx).
Where("conversation_id = ? And owner_user_id = ?", conversationID, ownerUserID).
Select("recv_msg_opt").
Find(&conversation).
Error,
"",
)
return int(conversation.RecvMsgOpt), utils.Wrap(c.db(ctx).Where("conversation_id = ? And owner_user_id = ?", conversationID, ownerUserID).Select("recv_msg_opt").Find(&conversation).Error, "")
}
func (c *ConversationGorm) GetAllConversationIDs(ctx context.Context) (conversationIDs []string, err error) {
return conversationIDs, utils.Wrap(
c.db(ctx).Distinct("conversation_id").Pluck("conversation_id", &conversationIDs).Error,
"",
)
return conversationIDs, utils.Wrap(c.db(ctx).Distinct("conversation_id").Pluck("conversation_id", &conversationIDs).Error, "")
}
func (c *ConversationGorm) GetUserAllHasReadSeqs(
ctx context.Context,
ownerUserID string,
) (hasReadSeqs map[string]int64, err error) {
var conversations []*relation.ConversationModel
err = utils.Wrap(
c.db(ctx).
Where("owner_user_id = ?", ownerUserID).
Select("conversation_id", "has_read_seq").
Find(&conversations).
Error,
"",
)
hasReadSeqs = make(map[string]int64, len(conversations))
// for _, conversation := range conversations {
// hasReadSeqs[conversation.ConversationID] = conversation.HasReadSeq
// }
return hasReadSeqs, err
func (c *ConversationGorm) GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (hasReadSeqs map[string]int64, err error) {
return nil, nil
}
func (c *ConversationGorm) GetConversationsByConversationID(
ctx context.Context,
conversationIDs []string,
) (conversations []*relation.ConversationModel, err error) {
return conversations, utils.Wrap(
c.db(ctx).Where("conversation_id IN (?)", conversationIDs).Find(&conversations).Error,
"",
)
func (c *ConversationGorm) GetConversationsByConversationID(ctx context.Context, conversationIDs []string) (conversations []*relation.ConversationModel, err error) {
return conversations, utils.Wrap(c.db(ctx).Where("conversation_id IN (?)", conversationIDs).Find(&conversations).Error, "")
}
func (c *ConversationGorm) GetConversationIDsNeedDestruct(ctx context.Context) (conversations []*relation.ConversationModel, err error) {
return conversations, utils.Wrap(c.db(ctx).Where("is_msg_destruct = 1 && UNIX_TIMESTAMP(NOW()) > (msg_destruct_time + UNIX_TIMESTAMP(latest_msg_destruct_time)) && msg_destruct_time != 0").Find(&conversations).Error, "")
}
pkg/common/db/relation/group_model.go
+33 -33
View File
@@ -1,27 +1,13 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package relation
import (
"context"
"gorm.io/gorm"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/ormutil"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"gorm.io/gorm"
"time"
)
var _ relation.GroupModelInterface = (*GroupGorm)(nil)
@@ -47,13 +33,7 @@ func (g *GroupGorm) UpdateMap(ctx context.Context, groupID string, args map[stri
}
func (g *GroupGorm) UpdateStatus(ctx context.Context, groupID string, status int32) (err error) {
return utils.Wrap(
g.DB.Where("group_id = ?", groupID).
Model(&relation.GroupModel{}).
Updates(map[string]any{"status": status}).
Error,
"",
)
return utils.Wrap(g.DB.Where("group_id = ?", groupID).Model(&relation.GroupModel{}).Updates(map[string]any{"status": status}).Error, "")
}
func (g *GroupGorm) Find(ctx context.Context, groupIDs []string) (groups []*relation.GroupModel, err error) {
@@ -65,17 +45,37 @@ func (g *GroupGorm) Take(ctx context.Context, groupID string) (group *relation.G
return group, utils.Wrap(g.DB.Where("group_id = ?", groupID).Take(group).Error, "")
}
func (g *GroupGorm) Search(
ctx context.Context,
keyword string,
pageNumber, showNumber int32,
) (total uint32, groups []*relation.GroupModel, err error) {
func (g *GroupGorm) Search(ctx context.Context, keyword string, pageNumber, showNumber int32) (total uint32, groups []*relation.GroupModel, err error) {
return ormutil.GormSearch[relation.GroupModel](g.DB, []string{"name"}, keyword, pageNumber, showNumber)
}
func (g *GroupGorm) GetGroupIDsByGroupType(ctx context.Context, groupType int) (groupIDs []string, err error) {
return groupIDs, utils.Wrap(
g.DB.Model(&relation.GroupModel{}).Where("group_type = ? ", groupType).Pluck("group_id", &groupIDs).Error,
"",
)
return groupIDs, utils.Wrap(g.DB.Model(&relation.GroupModel{}).Where("group_type = ? ", groupType).Pluck("group_id", &groupIDs).Error, "")
}
func (g *GroupGorm) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) {
db := g.db(ctx).Model(&relation.GroupModel{})
if before != nil {
db = db.Where("create_time < ?", before)
}
if err := db.Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}
func (g *GroupGorm) CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) {
var res []struct {
Date time.Time `gorm:"column:date"`
Count int64 `gorm:"column:count"`
}
err := g.db(ctx).Model(&relation.GroupModel{}).Select("DATE(create_time) AS date, count(1) AS count").Where("create_time >= ? and create_time < ?", start, end).Group("date").Find(&res).Error
if err != nil {
return nil, errs.Wrap(err)
}
v := make(map[string]int64)
for _, r := range res {
v[r.Date.Format("2006-01-02")] = r.Count
}
return v, nil
}
pkg/common/db/relation/mysql_init.go
+2 -2
View File
@@ -34,7 +34,7 @@ const (
maxRetry = 100 //number of retries
)
//newMysqlGormDB Initialize the database connection
// newMysqlGormDB Initialize the database connection
func newMysqlGormDB() (*gorm.DB, error) {
dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=true&loc=Local",
config.Config.Mysql.Username, config.Config.Mysql.Password, config.Config.Mysql.Address[0], "mysql")
@@ -84,7 +84,7 @@ func newMysqlGormDB() (*gorm.DB, error) {
return db, nil
}
//connectToDatabase Connection retry for mysql
// connectToDatabase Connection retry for mysql
func connectToDatabase(dsn string, maxRetry int) (*gorm.DB, error) {
var db *gorm.DB
var err error
pkg/common/db/relation/mysql_init_test.go
-38
View File
@@ -1,38 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package relation
import (
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"testing"
)
//TestNewGormDB Test the retry of sporadic errors and the direct exit of wrong password.
func TestNewGormDB(t *testing.T) {
err := config.InitConfig("config_folder_path")
if err != nil {
fmt.Println("config load error")
return
}
db, err := newMysqlGormDB()
if err != nil {
fmt.Println("password error")
return
}
if db != nil {
fmt.Println("success connect")
}
}
pkg/common/db/relation/object_hash_model.go
-66
View File
@@ -1,66 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package relation
import (
"context"
"gorm.io/gorm"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
)
type ObjectHashGorm struct {
*MetaDB
}
func NewObjectHash(db *gorm.DB) relation.ObjectHashModelInterface {
return &ObjectHashGorm{
NewMetaDB(db, &relation.ObjectHashModel{}),
}
}
func (o *ObjectHashGorm) NewTx(tx any) relation.ObjectHashModelInterface {
return &ObjectHashGorm{
NewMetaDB(tx.(*gorm.DB), &relation.ObjectHashModel{}),
}
}
func (o *ObjectHashGorm) Take(
ctx context.Context,
hash string,
engine string,
) (oh *relation.ObjectHashModel, err error) {
oh = &relation.ObjectHashModel{}
return oh, utils.Wrap1(o.DB.Where("hash = ? and engine = ?", hash, engine).Take(oh).Error)
}
func (o *ObjectHashGorm) Create(ctx context.Context, h []*relation.ObjectHashModel) (err error) {
return utils.Wrap1(o.DB.Create(h).Error)
}
func (o *ObjectHashGorm) DeleteNoCitation(
ctx context.Context,
engine string,
num int,
) (list []*relation.ObjectHashModel, err error) {
err = o.DB.Table(relation.ObjectHashModelTableName, "as h").Select("h.*").
Joins("LEFT JOIN "+relation.ObjectInfoModelTableName+" as i ON h.hash = i.hash").
Where("h.engine = ? AND i.hash IS NULL", engine).
Limit(num).
Find(&list).Error
return list, utils.Wrap1(err)
}
pkg/common/db/relation/object_info_model.go
-69
View File
@@ -1,69 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package relation
import (
"context"
"time"
"gorm.io/gorm"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
)
type ObjectInfoGorm struct {
*MetaDB
}
func NewObjectInfo(db *gorm.DB) relation.ObjectInfoModelInterface {
return &ObjectInfoGorm{
NewMetaDB(db, &relation.ObjectInfoModel{}),
}
}
func (o *ObjectInfoGorm) NewTx(tx any) relation.ObjectInfoModelInterface {
return &ObjectInfoGorm{
NewMetaDB(tx.(*gorm.DB), &relation.ObjectInfoModel{}),
}
}
func (o *ObjectInfoGorm) SetObject(ctx context.Context, obj *relation.ObjectInfoModel) (err error) {
if err := o.DB.WithContext(ctx).Where("name = ?", obj.Name).Delete(&relation.ObjectInfoModel{}).Error; err != nil {
return errs.Wrap(err)
}
return errs.Wrap(o.DB.WithContext(ctx).Create(obj).Error)
//return errs.Wrap(o.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
// if err := tx.Where("name = ?", obj.Name).Delete(&relation.ObjectInfoModel{}).Error; err != nil {
// return errs.Wrap(err)
// }
// return errs.Wrap(tx.Create(obj).Error)
//}))
}
func (o *ObjectInfoGorm) Take(ctx context.Context, name string) (info *relation.ObjectInfoModel, err error) {
info = &relation.ObjectInfoModel{}
return info, utils.Wrap1(o.DB.WithContext(ctx).Where("name = ?", name).Take(info).Error)
}
func (o *ObjectInfoGorm) DeleteExpiration(ctx context.Context, expiration time.Time) (err error) {
return utils.Wrap1(
o.DB.WithContext(ctx).
Where("expiration_time IS NOT NULL AND expiration_time <= ?", expiration).
Delete(&relation.ObjectInfoModel{}).
Error,
)
}
pkg/common/db/relation/object_model.go
+36
View File
@@ -0,0 +1,36 @@
package relation
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"gorm.io/gorm"
)
type ObjectInfoGorm struct {
*MetaDB
}
func NewObjectInfo(db *gorm.DB) relation.ObjectInfoModelInterface {
return &ObjectInfoGorm{
NewMetaDB(db, &relation.ObjectModel{}),
}
}
func (o *ObjectInfoGorm) NewTx(tx any) relation.ObjectInfoModelInterface {
return &ObjectInfoGorm{
NewMetaDB(tx.(*gorm.DB), &relation.ObjectModel{}),
}
}
func (o *ObjectInfoGorm) SetObject(ctx context.Context, obj *relation.ObjectModel) (err error) {
if err := o.DB.WithContext(ctx).Where("name = ?", obj.Name).FirstOrCreate(obj).Error; err != nil {
return errs.Wrap(err)
}
return nil
}
func (o *ObjectInfoGorm) Take(ctx context.Context, name string) (info *relation.ObjectModel, err error) {
info = &relation.ObjectModel{}
return info, errs.Wrap(o.DB.WithContext(ctx).Where("name = ?", name).Take(info).Error)
}
pkg/common/db/relation/object_put_model.go
-67
View File
@@ -1,67 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package relation
import (
"context"
"time"
"gorm.io/gorm"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
)
type ObjectPutGorm struct {
*MetaDB
}
func NewObjectPut(db *gorm.DB) relation.ObjectPutModelInterface {
return &ObjectPutGorm{
NewMetaDB(db, &relation.ObjectPutModel{}),
}
}
func (o *ObjectPutGorm) NewTx(tx any) relation.ObjectPutModelInterface {
return &ObjectPutGorm{
NewMetaDB(tx.(*gorm.DB), &relation.ObjectPutModel{}),
}
}
func (o *ObjectPutGorm) Create(ctx context.Context, m []*relation.ObjectPutModel) (err error) {
return utils.Wrap1(o.DB.Create(m).Error)
}
func (o *ObjectPutGorm) Take(ctx context.Context, putID string) (put *relation.ObjectPutModel, err error) {
put = &relation.ObjectPutModel{}
return put, utils.Wrap1(o.DB.Where("put_id = ?", putID).Take(put).Error)
}
func (o *ObjectPutGorm) SetCompleted(ctx context.Context, putID string) (err error) {
return utils.Wrap1(o.DB.Model(&relation.ObjectPutModel{}).Where("put_id = ?", putID).Update("complete", true).Error)
}
func (o *ObjectPutGorm) FindExpirationPut(
ctx context.Context,
expirationTime time.Time,
num int,
) (list []*relation.ObjectPutModel, err error) {
err = o.DB.Where("effective_time <= ?", expirationTime).Limit(num).Find(&list).Error
return list, utils.Wrap1(err)
}
func (o *ObjectPutGorm) DelPut(ctx context.Context, ids []string) (err error) {
return utils.Wrap1(o.DB.Where("put_id IN ?", ids).Delete(&relation.ObjectPutModel{}).Error)
}
pkg/common/db/relation/user_model.go
+10 -4
View File
@@ -41,7 +41,7 @@ func (u *UserGorm) Create(ctx context.Context, users []*relation.UserModel) (err
// 更新用户信息 零值
func (u *UserGorm) UpdateByMap(ctx context.Context, userID string, args map[string]interface{}) (err error) {
return utils.Wrap(u.db(ctx).Where("user_id = ?", userID).Updates(args).Error, "")
return utils.Wrap(u.db(ctx).Model(&relation.UserModel{}).Where("user_id = ?", userID).Updates(args).Error, "")
}
// 更新多个用户信息 非零值
@@ -94,9 +94,15 @@ func (u *UserGorm) GetUserGlobalRecvMsgOpt(ctx context.Context, userID string) (
return opt, err
}
func (u *UserGorm) CountTotal(ctx context.Context) (count int64, err error) {
err = u.db(ctx).Model(&relation.UserModel{}).Count(&count).Error
return count, errs.Wrap(err)
func (u *UserGorm) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) {
db := u.db(ctx).Model(&relation.UserModel{})
if before != nil {
db = db.Where("create_time < ?", before)
}
if err := db.Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}
func (u *UserGorm) CountRangeEverydayTotal(
pkg/common/db/s3/cont/consts.go
+9
View File
@@ -0,0 +1,9 @@
package cont
const (
hashPath = "openim/data/hash/"
tempPath = "openim/temp/"
UploadTypeMultipart = 1 // 分片上传
UploadTypePresigned = 2 // 预签名上传
partSeparator = ","
)
pkg/common/db/s3/cont/controller.go
+244
View File
@@ -0,0 +1,244 @@
package cont
import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/google/uuid"
"path"
"strings"
"time"
)
func New(impl s3.Interface) *Controller {
return &Controller{impl: impl}
}
type Controller struct {
impl s3.Interface
}
func (c *Controller) HashPath(md5 string) string {
return path.Join(hashPath, md5)
}
func (c *Controller) NowPath() string {
now := time.Now()
return path.Join(
fmt.Sprintf("%04d", now.Year()),
fmt.Sprintf("%02d", now.Month()),
fmt.Sprintf("%02d", now.Day()),
fmt.Sprintf("%02d", now.Hour()),
fmt.Sprintf("%02d", now.Minute()),
fmt.Sprintf("%02d", now.Second()),
)
}
func (c *Controller) UUID() string {
id := uuid.New()
return hex.EncodeToString(id[:])
}
func (c *Controller) PartSize(ctx context.Context, size int64) (int64, error) {
return c.impl.PartSize(ctx, size)
}
func (c *Controller) PartLimit() *s3.PartLimit {
return c.impl.PartLimit()
}
func (c *Controller) GetHashObject(ctx context.Context, hash string) (*s3.ObjectInfo, error) {
return c.impl.StatObject(ctx, c.HashPath(hash))
}
func (c *Controller) InitiateUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*InitiateUploadResult, error) {
defer log.ZDebug(ctx, "return")
if size < 0 {
return nil, errors.New("invalid size")
}
if hashBytes, err := hex.DecodeString(hash); err != nil {
return nil, err
} else if len(hashBytes) != md5.Size {
return nil, errors.New("invalid md5")
}
partSize, err := c.impl.PartSize(ctx, size)
if err != nil {
return nil, err
}
partNumber := int(size / partSize)
if size%partSize > 0 {
partNumber++
}
if maxParts > 0 && partNumber > 0 && partNumber < maxParts {
return nil, errors.New(fmt.Sprintf("too many parts: %d", partNumber))
}
if info, err := c.impl.StatObject(ctx, c.HashPath(hash)); err == nil {
return nil, &HashAlreadyExistsError{Object: info}
} else if !c.impl.IsNotFound(err) {
return nil, err
}
if size <= partSize {
// 预签名上传
key := path.Join(tempPath, c.NowPath(), fmt.Sprintf("%s_%d_%s.presigned", hash, size, c.UUID()))
rawURL, err := c.impl.PresignedPutObject(ctx, key, expire)
if err != nil {
return nil, err
}
return &InitiateUploadResult{
UploadID: newMultipartUploadID(multipartUploadID{
Type: UploadTypePresigned,
ID: "",
Key: key,
Size: size,
Hash: hash,
}),
PartSize: partSize,
Sign: &s3.AuthSignResult{
Parts: []s3.SignPart{
{
PartNumber: 1,
URL: rawURL,
},
},
},
}, nil
} else {
// 分片上传
upload, err := c.impl.InitiateMultipartUpload(ctx, c.HashPath(hash))
if err != nil {
return nil, err
}
if maxParts < 0 {
maxParts = partNumber
}
var authSign *s3.AuthSignResult
if maxParts > 0 {
partNumbers := make([]int, partNumber)
for i := 0; i < maxParts; i++ {
partNumbers[i] = i + 1
}
authSign, err = c.impl.AuthSign(ctx, upload.UploadID, upload.Key, time.Hour*24, partNumbers)
if err != nil {
return nil, err
}
}
return &InitiateUploadResult{
UploadID: newMultipartUploadID(multipartUploadID{
Type: UploadTypeMultipart,
ID: upload.UploadID,
Key: upload.Key,
Size: size,
Hash: hash,
}),
PartSize: partSize,
Sign: authSign,
}, nil
}
}
func (c *Controller) CompleteUpload(ctx context.Context, uploadID string, partHashs []string) (*UploadResult, error) {
defer log.ZDebug(ctx, "return")
upload, err := parseMultipartUploadID(uploadID)
if err != nil {
return nil, err
}
if md5Sum := md5.Sum([]byte(strings.Join(partHashs, partSeparator))); hex.EncodeToString(md5Sum[:]) != upload.Hash {
fmt.Println("CompleteUpload sum:", hex.EncodeToString(md5Sum[:]), "upload hash:", upload.Hash)
return nil, errors.New("md5 mismatching")
}
if info, err := c.impl.StatObject(ctx, c.HashPath(upload.Hash)); err == nil {
return &UploadResult{
Key: info.Key,
Size: info.Size,
Hash: info.ETag,
}, nil
} else if !c.impl.IsNotFound(err) {
return nil, err
}
cleanObject := make(map[string]struct{})
defer func() {
for key := range cleanObject {
_ = c.impl.DeleteObject(ctx, key)
}
}()
var targetKey string
switch upload.Type {
case UploadTypeMultipart:
parts := make([]s3.Part, len(partHashs))
for i, part := range partHashs {
parts[i] = s3.Part{
PartNumber: i + 1,
ETag: part,
}
}
// todo: 验证大小
result, err := c.impl.CompleteMultipartUpload(ctx, upload.ID, upload.Key, parts)
if err != nil {
return nil, err
}
targetKey = result.Key
case UploadTypePresigned:
uploadInfo, err := c.impl.StatObject(ctx, upload.Key)
if err != nil {
return nil, err
}
cleanObject[uploadInfo.Key] = struct{}{}
if uploadInfo.Size != upload.Size {
return nil, errors.New("upload size mismatching")
}
md5Sum := md5.Sum([]byte(strings.Join([]string{uploadInfo.ETag}, partSeparator)))
if md5val := hex.EncodeToString(md5Sum[:]); md5val != upload.Hash {
return nil, errs.ErrArgs.Wrap(fmt.Sprintf("md5 mismatching %s != %s", md5val, upload.Hash))
}
// 防止在这个时候，并发操作，导致文件被覆盖
copyInfo, err := c.impl.CopyObject(ctx, uploadInfo.Key, upload.Key+"."+c.UUID())
if err != nil {
return nil, err
}
cleanObject[copyInfo.Key] = struct{}{}
if copyInfo.ETag != uploadInfo.ETag {
return nil, errors.New("[concurrency]copy md5 mismatching")
}
hashCopyInfo, err := c.impl.CopyObject(ctx, copyInfo.Key, c.HashPath(upload.Hash))
if err != nil {
return nil, err
}
log.ZInfo(ctx, "hashCopyInfo", "value", fmt.Sprintf("%+v", hashCopyInfo))
targetKey = hashCopyInfo.Key
default:
return nil, errors.New("invalid upload id type")
}
return &UploadResult{
Key: targetKey,
Size: upload.Size,
Hash: upload.Hash,
}, nil
}
func (c *Controller) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) {
upload, err := parseMultipartUploadID(uploadID)
if err != nil {
return nil, err
}
switch upload.Type {
case UploadTypeMultipart:
return c.impl.AuthSign(ctx, upload.ID, upload.Key, time.Hour*24, partNumbers)
case UploadTypePresigned:
return nil, errors.New("presigned id not support auth sign")
default:
return nil, errors.New("invalid upload id type")
}
}
func (c *Controller) IsNotFound(err error) bool {
return c.impl.IsNotFound(err)
}
func (c *Controller) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
return c.impl.AccessURL(ctx, name, expire, opt)
}
pkg/common/db/s3/cont/error.go
+14
View File
@@ -0,0 +1,14 @@
package cont
import (
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
)
type HashAlreadyExistsError struct {
Object *s3.ObjectInfo
}
func (e *HashAlreadyExistsError) Error() string {
return fmt.Sprintf("hash already exists: %s", e.Object.Key)
}
pkg/common/db/s3/cont/id.go
+35
View File
@@ -0,0 +1,35 @@
package cont
import (
"encoding/base64"
"encoding/json"
"fmt"
)
type multipartUploadID struct {
Type int `json:"a,omitempty"`
ID string `json:"b,omitempty"`
Key string `json:"c,omitempty"`
Size int64 `json:"d,omitempty"`
Hash string `json:"e,omitempty"`
}
func newMultipartUploadID(id multipartUploadID) string {
data, err := json.Marshal(id)
if err != nil {
panic(err)
}
return base64.StdEncoding.EncodeToString(data)
}
func parseMultipartUploadID(id string) (*multipartUploadID, error) {
data, err := base64.StdEncoding.DecodeString(id)
if err != nil {
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
}
var upload multipartUploadID
if err := json.Unmarshal(data, &upload); err != nil {
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
}
return &upload, nil
}
pkg/common/db/s3/cont/structs.go
+15
View File
@@ -0,0 +1,15 @@
package cont
import "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
type InitiateUploadResult struct {
UploadID string `json:"uploadID"` // 上传ID
PartSize int64 `json:"partSize"` // 分片大小
Sign *s3.AuthSignResult `json:"sign"` // 分片信息
}
type UploadResult struct {
Hash string `json:"hash"`
Size int64 `json:"size"`
Key string `json:"key"`
}
pkg/common/db/s3/cos/cos.go
+254
View File
@@ -0,0 +1,254 @@
package cos
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/tencentyun/cos-go-sdk-v5"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
const (
minPartSize = 1024 * 1024 * 1 // 1MB
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
maxNumSize = 1000
)
func NewCos() (s3.Interface, error) {
conf := config.Config.Object.Cos
u, err := url.Parse(conf.BucketURL)
if err != nil {
panic(err)
}
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: conf.SecretID,
SecretKey: conf.SecretKey,
SessionToken: conf.SessionToken,
},
})
return &Cos{
copyURL: u.Host + "/",
client: client,
credential: client.GetCredential(),
}, nil
}
type Cos struct {
copyURL string
client *cos.Client
credential *cos.Credential
}
func (c *Cos) Engine() string {
return "tencent-cos"
}
func (c *Cos) PartLimit() *s3.PartLimit {
return &s3.PartLimit{
MinPartSize: minPartSize,
MaxPartSize: maxPartSize,
MaxNumSize: maxNumSize,
}
}
func (c *Cos) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
result, _, err := c.client.Object.InitiateMultipartUpload(ctx, name, nil)
if err != nil {
return nil, err
}
return &s3.InitiateMultipartUploadResult{
UploadID: result.UploadID,
Bucket: result.Bucket,
Key: result.Key,
}, nil
}
func (c *Cos) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
opts := &cos.CompleteMultipartUploadOptions{
Parts: make([]cos.Object, len(parts)),
}
for i, part := range parts {
opts.Parts[i] = cos.Object{
PartNumber: part.PartNumber,
ETag: strings.ReplaceAll(part.ETag, `"`, ``),
}
}
result, _, err := c.client.Object.CompleteMultipartUpload(ctx, name, uploadID, opts)
if err != nil {
return nil, err
}
return &s3.CompleteMultipartUploadResult{
Location: result.Location,
Bucket: result.Bucket,
Key: result.Key,
ETag: result.ETag,
}, nil
}
func (c *Cos) PartSize(ctx context.Context, size int64) (int64, error) {
if size <= 0 {
return 0, errors.New("size must be greater than 0")
}
if size > maxPartSize*maxNumSize {
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
}
if size <= minPartSize*maxNumSize {
return minPartSize, nil
}
partSize := size / maxNumSize
if size%maxNumSize != 0 {
partSize++
}
return partSize, nil
}
func (c *Cos) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
result := s3.AuthSignResult{
URL: c.client.BaseURL.BucketURL.String() + "/" + cos.EncodeURIComponent(name),
Query: url.Values{"uploadId": {uploadID}},
Header: make(http.Header),
Parts: make([]s3.SignPart, len(partNumbers)),
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, result.URL, nil)
if err != nil {
return nil, err
}
cos.AddAuthorizationHeader(c.credential.SecretID, c.credential.SecretKey, c.credential.SessionToken, req, cos.NewAuthTime(expire))
result.Header = req.Header
for i, partNumber := range partNumbers {
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
}
}
return &result, nil
}
func (c *Cos) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodPut, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
if err != nil {
return "", err
}
return rawURL.String(), nil
}
func (c *Cos) DeleteObject(ctx context.Context, name string) error {
_, err := c.client.Object.Delete(ctx, name)
return err
}
func (c *Cos) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
if name != "" && name[0] == '/' {
name = name[1:]
}
info, err := c.client.Object.Head(ctx, name, nil)
if err != nil {
return nil, err
}
res := &s3.ObjectInfo{Key: name}
if res.ETag = strings.ToLower(strings.ReplaceAll(info.Header.Get("ETag"), `"`, "")); res.ETag == "" {
return nil, errors.New("StatObject etag not found")
}
if contentLengthStr := info.Header.Get("Content-Length"); contentLengthStr == "" {
return nil, errors.New("StatObject content-length not found")
} else {
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return nil, fmt.Errorf("StatObject content-length parse error: %w", err)
}
if res.Size < 0 {
return nil, errors.New("StatObject content-length must be greater than 0")
}
}
if lastModified := info.Header.Get("Last-Modified"); lastModified == "" {
return nil, errors.New("StatObject last-modified not found")
} else {
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
if err != nil {
return nil, fmt.Errorf("StatObject last-modified parse error: %w", err)
}
}
return res, nil
}
func (c *Cos) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
sourceURL := c.copyURL + src
result, _, err := c.client.Object.Copy(ctx, dst, sourceURL, nil)
if err != nil {
return nil, err
}
return &s3.CopyObjectInfo{
Key: dst,
ETag: strings.ReplaceAll(result.ETag, `"`, ``),
}, nil
}
func (c *Cos) IsNotFound(err error) bool {
switch e := err.(type) {
case *cos.ErrorResponse:
return e.Response.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (c *Cos) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
_, err := c.client.Object.AbortMultipartUpload(ctx, name, uploadID)
return err
}
func (c *Cos) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
result, _, err := c.client.Object.ListParts(ctx, name, uploadID, &cos.ObjectListPartsOptions{
MaxParts: strconv.Itoa(maxParts),
PartNumberMarker: strconv.Itoa(partNumberMarker),
})
if err != nil {
return nil, err
}
res := &s3.ListUploadedPartsResult{
Key: result.Key,
UploadID: result.UploadID,
UploadedParts: make([]s3.UploadedPart, len(result.Parts)),
}
res.MaxParts, _ = strconv.Atoi(result.MaxParts)
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
for i, part := range result.Parts {
lastModified, _ := time.Parse(http.TimeFormat, part.LastModified)
res.UploadedParts[i] = s3.UploadedPart{
PartNumber: part.PartNumber,
LastModified: lastModified,
ETag: part.ETag,
Size: part.Size,
}
}
return res, nil
}
func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
//reqParams := make(url.Values)
//if opt != nil {
// if opt.ContentType != "" {
// reqParams.Set("Content-Type", opt.ContentType)
// }
// if opt.ContentDisposition != "" {
// reqParams.Set("Content-Disposition", opt.ContentDisposition)
// }
//}
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
expire = time.Second
}
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
if err != nil {
return "", err
}
return rawURL.String(), nil
}
pkg/common/db/s3/minio/minio.go
+250
View File
@@ -0,0 +1,250 @@
package minio
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/signer"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
const (
unsignedPayload = "UNSIGNED-PAYLOAD"
)
const (
minPartSize = 1024 * 1024 * 5 // 1MB
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
maxNumSize = 10000
)
func NewMinio() (s3.Interface, error) {
conf := config.Config.Object.Minio
u, err := url.Parse(conf.Endpoint)
if err != nil {
return nil, err
}
opts := &minio.Options{
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken),
Secure: u.Scheme == "https",
}
client, err := minio.New(u.Host, opts)
if err != nil {
return nil, err
}
return &Minio{
bucket: conf.Bucket,
bucketURL: conf.Endpoint + "/" + conf.Bucket + "/",
opts: opts,
core: &minio.Core{Client: client},
}, nil
}
type Minio struct {
bucket string
bucketURL string
opts *minio.Options
core *minio.Core
}
func (m *Minio) Engine() string {
return "minio"
}
func (m *Minio) PartLimit() *s3.PartLimit {
return &s3.PartLimit{
MinPartSize: minPartSize,
MaxPartSize: maxPartSize,
MaxNumSize: maxNumSize,
}
}
func (m *Minio) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
uploadID, err := m.core.NewMultipartUpload(ctx, m.bucket, name, minio.PutObjectOptions{})
if err != nil {
return nil, err
}
return &s3.InitiateMultipartUploadResult{
Bucket: m.bucket,
Key: name,
UploadID: uploadID,
}, nil
}
func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
minioParts := make([]minio.CompletePart, len(parts))
for i, part := range parts {
minioParts[i] = minio.CompletePart{
PartNumber: part.PartNumber,
ETag: strings.ToLower(part.ETag),
}
}
upload, err := m.core.CompleteMultipartUpload(ctx, m.bucket, name, uploadID, minioParts, minio.PutObjectOptions{})
if err != nil {
return nil, err
}
return &s3.CompleteMultipartUploadResult{
Location: upload.Location,
Bucket: upload.Bucket,
Key: upload.Key,
ETag: strings.ToLower(upload.ETag),
}, nil
}
func (m *Minio) PartSize(ctx context.Context, size int64) (int64, error) {
if size <= 0 {
return 0, errors.New("size must be greater than 0")
}
if size > maxPartSize*maxNumSize {
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
}
if size <= minPartSize*maxNumSize {
return minPartSize, nil
}
partSize := size / maxNumSize
if size%maxNumSize != 0 {
partSize++
}
return partSize, nil
}
func (m *Minio) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
creds, err := m.opts.Creds.Get()
if err != nil {
return nil, err
}
result := s3.AuthSignResult{
URL: m.bucketURL + name,
Query: url.Values{"uploadId": {uploadID}},
Parts: make([]s3.SignPart, len(partNumbers)),
}
for i, partNumber := range partNumbers {
rawURL := result.URL + "?partNumber=" + strconv.Itoa(partNumber) + "&uploadId=" + uploadID
request, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, nil)
if err != nil {
return nil, err
}
request.Header.Set("X-Amz-Content-Sha256", unsignedPayload)
request = signer.SignV4Trailer(*request, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken, "us-east-1", nil)
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
URL: request.URL.String(),
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
Header: request.Header,
}
}
return &result, nil
}
func (m *Minio) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
rawURL, err := m.core.Client.PresignedPutObject(ctx, m.bucket, name, expire)
if err != nil {
return "", err
}
return rawURL.String(), nil
}
func (m *Minio) DeleteObject(ctx context.Context, name string) error {
return m.core.Client.RemoveObject(ctx, m.bucket, name, minio.RemoveObjectOptions{})
}
func (m *Minio) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
info, err := m.core.Client.StatObject(ctx, m.bucket, name, minio.StatObjectOptions{})
if err != nil {
return nil, err
}
return &s3.ObjectInfo{
ETag: strings.ToLower(info.ETag),
Key: info.Key,
Size: info.Size,
LastModified: info.LastModified,
}, nil
}
func (m *Minio) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
result, err := m.core.Client.CopyObject(ctx, minio.CopyDestOptions{
Bucket: m.bucket,
Object: dst,
}, minio.CopySrcOptions{
Bucket: m.bucket,
Object: src,
})
if err != nil {
return nil, err
}
return &s3.CopyObjectInfo{
Key: dst,
ETag: strings.ToLower(result.ETag),
}, nil
}
func (m *Minio) IsNotFound(err error) bool {
if err == nil {
return false
}
switch e := err.(type) {
case minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
case *minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (m *Minio) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
return m.core.AbortMultipartUpload(ctx, m.bucket, name, uploadID)
}
func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
result, err := m.core.ListObjectParts(ctx, m.bucket, name, uploadID, partNumberMarker, maxParts)
if err != nil {
return nil, err
}
res := &s3.ListUploadedPartsResult{
Key: result.Key,
UploadID: result.UploadID,
MaxParts: result.MaxParts,
NextPartNumberMarker: result.NextPartNumberMarker,
UploadedParts: make([]s3.UploadedPart, len(result.ObjectParts)),
}
for i, part := range result.ObjectParts {
res.UploadedParts[i] = s3.UploadedPart{
PartNumber: part.PartNumber,
LastModified: part.LastModified,
ETag: part.ETag,
Size: part.Size,
}
}
return res, nil
}
func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
//reqParams := make(url.Values)
//if opt != nil {
// if opt.ContentType != "" {
// reqParams.Set("Content-Type", opt.ContentType)
// }
// if opt.ContentDisposition != "" {
// reqParams.Set("Content-Disposition", opt.ContentDisposition)
// }
//}
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
expire = time.Second
}
u, err := m.core.Client.PresignedGetObject(ctx, m.bucket, name, expire, nil)
if err != nil {
return "", err
}
return u.String(), nil
}
pkg/common/db/s3/oss/oss.go
+259
View File
@@ -0,0 +1,259 @@
package oss
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
const (
minPartSize = 1024 * 1024 * 1 // 1MB
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
maxNumSize = 10000
)
func NewOSS() (s3.Interface, error) {
conf := config.Config.Object.Oss
if conf.BucketURL == "" {
return nil, errors.New("bucket url is empty")
}
client, err := oss.New(conf.Endpoint, conf.AccessKeyID, conf.AccessKeySecret)
if err != nil {
return nil, err
}
bucket, err := client.Bucket(conf.Bucket)
if err != nil {
return nil, err
}
if conf.BucketURL[len(conf.BucketURL)-1] != '/' {
conf.BucketURL += "/"
}
return &OSS{
bucketURL: conf.BucketURL,
bucket: bucket,
credentials: client.Config.GetCredentials(),
}, nil
}
type OSS struct {
bucketURL string
bucket *oss.Bucket
credentials oss.Credentials
}
func (o *OSS) Engine() string {
return "ali-oss"
}
func (o *OSS) PartLimit() *s3.PartLimit {
return &s3.PartLimit{
MinPartSize: minPartSize,
MaxPartSize: maxPartSize,
MaxNumSize: maxNumSize,
}
}
func (o *OSS) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
result, err := o.bucket.InitiateMultipartUpload(name)
if err != nil {
return nil, err
}
return &s3.InitiateMultipartUploadResult{
UploadID: result.UploadID,
Bucket: result.Bucket,
Key: result.Key,
}, nil
}
func (o *OSS) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
ossParts := make([]oss.UploadPart, len(parts))
for i, part := range parts {
ossParts[i] = oss.UploadPart{
PartNumber: part.PartNumber,
ETag: strings.ToUpper(part.ETag),
}
}
result, err := o.bucket.CompleteMultipartUpload(oss.InitiateMultipartUploadResult{
UploadID: uploadID,
Bucket: o.bucket.BucketName,
Key: name,
}, ossParts)
if err != nil {
return nil, err
}
return &s3.CompleteMultipartUploadResult{
Location: result.Location,
Bucket: result.Bucket,
Key: result.Key,
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
}, nil
}
func (o *OSS) PartSize(ctx context.Context, size int64) (int64, error) {
if size <= 0 {
return 0, errors.New("size must be greater than 0")
}
if size > maxPartSize*maxNumSize {
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
}
if size <= minPartSize*maxNumSize {
return minPartSize, nil
}
partSize := size / maxNumSize
if size%maxNumSize != 0 {
partSize++
}
return partSize, nil
}
func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
result := s3.AuthSignResult{
URL: o.bucketURL + name,
Query: url.Values{"uploadId": {uploadID}},
Header: make(http.Header),
Parts: make([]s3.SignPart, len(partNumbers)),
}
for i, partNumber := range partNumbers {
rawURL := fmt.Sprintf(`%s%s?partNumber=%d&uploadId=%s`, o.bucketURL, name, partNumber, uploadID)
request, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, nil)
if err != nil {
return nil, err
}
if o.credentials.GetSecurityToken() != "" {
request.Header.Set(oss.HTTPHeaderOssSecurityToken, o.credentials.GetSecurityToken())
}
request.Header.Set(oss.HTTPHeaderHost, request.Host)
request.Header.Set(oss.HTTPHeaderDate, time.Now().UTC().Format(http.TimeFormat))
authorization := fmt.Sprintf(`OSS %s:%s`, o.credentials.GetAccessKeyID(), o.getSignedStr(request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID), o.credentials.GetAccessKeySecret()))
request.Header.Set(oss.HTTPHeaderAuthorization, authorization)
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
URL: request.URL.String(),
Header: request.Header,
}
}
return &result, nil
}
func (o *OSS) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
return o.bucket.SignURL(name, http.MethodPut, int64(expire/time.Second))
}
func (o *OSS) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
header, err := o.bucket.GetObjectMeta(name)
if err != nil {
return nil, err
}
res := &s3.ObjectInfo{Key: name}
if res.ETag = strings.ToLower(strings.ReplaceAll(header.Get("ETag"), `"`, ``)); res.ETag == "" {
return nil, errors.New("StatObject etag not found")
}
if contentLengthStr := header.Get("Content-Length"); contentLengthStr == "" {
return nil, errors.New("StatObject content-length not found")
} else {
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return nil, fmt.Errorf("StatObject content-length parse error: %w", err)
}
if res.Size < 0 {
return nil, errors.New("StatObject content-length must be greater than 0")
}
}
if lastModified := header.Get("Last-Modified"); lastModified == "" {
return nil, errors.New("StatObject last-modified not found")
} else {
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
if err != nil {
return nil, fmt.Errorf("StatObject last-modified parse error: %w", err)
}
}
return res, nil
}
func (o *OSS) DeleteObject(ctx context.Context, name string) error {
return o.bucket.DeleteObject(name)
}
func (o *OSS) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
result, err := o.bucket.CopyObject(src, dst)
if err != nil {
return nil, err
}
return &s3.CopyObjectInfo{
Key: dst,
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
}, nil
}
func (o *OSS) IsNotFound(err error) bool {
switch e := err.(type) {
case oss.ServiceError:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
case *oss.ServiceError:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (o *OSS) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
return o.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{
UploadID: uploadID,
Key: name,
Bucket: o.bucket.BucketName,
})
}
func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
result, err := o.bucket.ListUploadedParts(oss.InitiateMultipartUploadResult{
UploadID: uploadID,
Key: name,
Bucket: o.bucket.BucketName,
}, oss.MaxUploads(100), oss.MaxParts(maxParts), oss.PartNumberMarker(partNumberMarker))
if err != nil {
return nil, err
}
res := &s3.ListUploadedPartsResult{
Key: result.Key,
UploadID: result.UploadID,
MaxParts: result.MaxParts,
UploadedParts: make([]s3.UploadedPart, len(result.UploadedParts)),
}
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
for i, part := range result.UploadedParts {
res.UploadedParts[i] = s3.UploadedPart{
PartNumber: part.PartNumber,
LastModified: part.LastModified,
ETag: part.ETag,
Size: int64(part.Size),
}
}
return res, nil
}
func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
//var opts []oss.Option
//if opt != nil {
// if opt.ContentType != "" {
// opts = append(opts, oss.ContentType(opt.ContentType))
// }
// if opt.ContentDisposition != "" {
// opts = append(opts, oss.ContentDisposition(opt.ContentDisposition))
// }
//}
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
expire = time.Second
}
return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second))
}
pkg/common/db/s3/oss/sign.go
+81
View File
@@ -0,0 +1,81 @@
package oss
import (
"crypto/hmac"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"hash"
"io"
"net/http"
"sort"
"strings"
)
func (o *OSS) getAdditionalHeaderKeys(req *http.Request) ([]string, map[string]string) {
var keysList []string
keysMap := make(map[string]string)
srcKeys := make(map[string]string)
for k := range req.Header {
srcKeys[strings.ToLower(k)] = ""
}
for _, v := range o.bucket.Client.Config.AdditionalHeaders {
if _, ok := srcKeys[strings.ToLower(v)]; ok {
keysMap[strings.ToLower(v)] = ""
}
}
for k := range keysMap {
keysList = append(keysList, k)
}
sort.Strings(keysList)
return keysList, keysMap
}
func (o *OSS) getSignedStr(req *http.Request, canonicalizedResource string, keySecret string) string {
// Find out the "x-oss-"'s address in header of the request
ossHeadersMap := make(map[string]string)
additionalList, additionalMap := o.getAdditionalHeaderKeys(req)
for k, v := range req.Header {
if strings.HasPrefix(strings.ToLower(k), "x-oss-") {
ossHeadersMap[strings.ToLower(k)] = v[0]
} else if o.bucket.Client.Config.AuthVersion == oss.AuthV2 {
if _, ok := additionalMap[strings.ToLower(k)]; ok {
ossHeadersMap[strings.ToLower(k)] = v[0]
}
}
}
hs := newHeaderSorter(ossHeadersMap)
// Sort the ossHeadersMap by the ascending order
hs.Sort()
// Get the canonicalizedOSSHeaders
canonicalizedOSSHeaders := ""
for i := range hs.Keys {
canonicalizedOSSHeaders += hs.Keys[i] + ":" + hs.Vals[i] + "\n"
}
// Give other parameters values
// when sign URL, date is expires
date := req.Header.Get(oss.HTTPHeaderDate)
contentType := req.Header.Get(oss.HTTPHeaderContentType)
contentMd5 := req.Header.Get(oss.HTTPHeaderContentMD5)
// default is v1 signature
signStr := req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + canonicalizedResource
h := hmac.New(func() hash.Hash { return sha1.New() }, []byte(keySecret))
// v2 signature
if o.bucket.Client.Config.AuthVersion == oss.AuthV2 {
signStr = req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + strings.Join(additionalList, ";") + "\n" + canonicalizedResource
h = hmac.New(func() hash.Hash { return sha256.New() }, []byte(keySecret))
}
_, _ = io.WriteString(h, signStr)
signedStr := base64.StdEncoding.EncodeToString(h.Sum(nil))
return signedStr
}
pkg/common/db/s3/oss/sort.go
+47
View File
@@ -0,0 +1,47 @@
package oss
import (
"bytes"
"sort"
)
// headerSorter defines the key-value structure for storing the sorted data in signHeader.
type headerSorter struct {
Keys []string
Vals []string
}
// newHeaderSorter is an additional function for function SignHeader.
func newHeaderSorter(m map[string]string) *headerSorter {
hs := &headerSorter{
Keys: make([]string, 0, len(m)),
Vals: make([]string, 0, len(m)),
}
for k, v := range m {
hs.Keys = append(hs.Keys, k)
hs.Vals = append(hs.Vals, v)
}
return hs
}
// Sort is an additional function for function SignHeader.
func (hs *headerSorter) Sort() {
sort.Sort(hs)
}
// Len is an additional function for function SignHeader.
func (hs *headerSorter) Len() int {
return len(hs.Vals)
}
// Less is an additional function for function SignHeader.
func (hs *headerSorter) Less(i, j int) bool {
return bytes.Compare([]byte(hs.Keys[i]), []byte(hs.Keys[j])) < 0
}
// Swap is an additional function for function SignHeader.
func (hs *headerSorter) Swap(i, j int) {
hs.Vals[i], hs.Vals[j] = hs.Vals[j], hs.Vals[i]
hs.Keys[i], hs.Keys[j] = hs.Keys[j], hs.Keys[i]
}
pkg/common/db/s3/s3.go
+134
View File
@@ -0,0 +1,134 @@
package s3
import (
"context"
"net/http"
"net/url"
"time"
)
type PartLimit struct {
MinPartSize int64 `json:"minPartSize"`
MaxPartSize int64 `json:"maxPartSize"`
MaxNumSize int `json:"maxNumSize"`
}
type InitiateMultipartUploadResult struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
UploadID string `json:"uploadID"`
}
type MultipartUploadRequest struct {
UploadID string `json:"uploadId"`
Bucket string `json:"bucket"`
Key string `json:"key"`
Method string `json:"method"`
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
PartKey string `json:"partKey"`
PartSize int64 `json:"partSize"`
FirstPart int `json:"firstPart"`
}
type Part struct {
PartNumber int `json:"partNumber"`
ETag string `json:"etag"`
}
type CompleteMultipartUploadResult struct {
Location string `json:"location"`
Bucket string `json:"bucket"`
Key string `json:"key"`
ETag string `json:"etag"`
}
type SignResult struct {
Parts []SignPart `json:"parts"`
}
type ObjectInfo struct {
ETag string `json:"etag"`
Key string `json:"name"`
Size int64 `json:"size"`
LastModified time.Time `json:"lastModified"`
}
type CopyObjectInfo struct {
Key string `json:"name"`
ETag string `json:"etag"`
}
type SignPart struct {
PartNumber int `json:"partNumber"`
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
}
type AuthSignResult struct {
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
Parts []SignPart `json:"parts"`
}
type InitiateUpload struct {
UploadID string `json:"uploadId"`
Bucket string `json:"bucket"`
Key string `json:"key"`
Method string `json:"method"`
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
PartKey string `json:"partKey"`
PartSize int64 `json:"partSize"`
FirstPart int `json:"firstPart"`
}
type UploadedPart struct {
PartNumber int `json:"partNumber"`
LastModified time.Time `json:"lastModified"`
ETag string `json:"etag"`
Size int64 `json:"size"`
}
type ListUploadedPartsResult struct {
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
NextPartNumberMarker int `xml:"NextPartNumberMarker"`
MaxParts int `xml:"MaxParts"`
UploadedParts []UploadedPart `xml:"Part"`
}
type AccessURLOption struct {
ContentType string `json:"contentType"`
ContentDisposition string `json:"contentDisposition"`
}
type Interface interface {
Engine() string
PartLimit() *PartLimit
InitiateMultipartUpload(ctx context.Context, name string) (*InitiateMultipartUploadResult, error)
CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []Part) (*CompleteMultipartUploadResult, error)
PartSize(ctx context.Context, size int64) (int64, error)
AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*AuthSignResult, error)
PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error)
DeleteObject(ctx context.Context, name string) error
CopyObject(ctx context.Context, src string, dst string) (*CopyObjectInfo, error)
StatObject(ctx context.Context, name string) (*ObjectInfo, error)
IsNotFound(err error) bool
AbortMultipartUpload(ctx context.Context, uploadID string, name string) error
ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*ListUploadedPartsResult, error)
AccessURL(ctx context.Context, name string, expire time.Duration, opt *AccessURLOption) (string, error)
}
pkg/common/db/table/relation/conversation.go
+26 -45
View File
@@ -1,40 +1,33 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package relation
import "context"
import (
"context"
"time"
)
const (
conversationModelTableName = "conversations"
)
type ConversationModel struct {
OwnerUserID string `gorm:"column:owner_user_id;primary_key;type:char(128)" json:"OwnerUserID"`
ConversationID string `gorm:"column:conversation_id;primary_key;type:char(128)" json:"conversationID"`
ConversationType int32 `gorm:"column:conversation_type" json:"conversationType"`
UserID string `gorm:"column:user_id;type:char(64)" json:"userID"`
GroupID string `gorm:"column:group_id;type:char(128)" json:"groupID"`
RecvMsgOpt int32 `gorm:"column:recv_msg_opt" json:"recvMsgOpt"`
IsPinned bool `gorm:"column:is_pinned" json:"isPinned"`
IsPrivateChat bool `gorm:"column:is_private_chat" json:"isPrivateChat"`
BurnDuration int32 `gorm:"column:burn_duration;default:30" json:"burnDuration"`
GroupAtType int32 `gorm:"column:group_at_type" json:"groupAtType"`
AttachedInfo string `gorm:"column:attached_info;type:varchar(1024)" json:"attachedInfo"`
Ex string `gorm:"column:ex;type:varchar(1024)" json:"ex"`
MaxSeq int64 `gorm:"column:max_seq" json:"maxSeq"`
MinSeq int64 `gorm:"column:min_seq" json:"minSeq"`
OwnerUserID string `gorm:"column:owner_user_id;primary_key;type:char(128)" json:"OwnerUserID"`
ConversationID string `gorm:"column:conversation_id;primary_key;type:char(128)" json:"conversationID"`
ConversationType int32 `gorm:"column:conversation_type" json:"conversationType"`
UserID string `gorm:"column:user_id;type:char(64)" json:"userID"`
GroupID string `gorm:"column:group_id;type:char(128)" json:"groupID"`
RecvMsgOpt int32 `gorm:"column:recv_msg_opt" json:"recvMsgOpt"`
IsPinned bool `gorm:"column:is_pinned" json:"isPinned"`
IsPrivateChat bool `gorm:"column:is_private_chat" json:"isPrivateChat"`
BurnDuration int32 `gorm:"column:burn_duration;default:30" json:"burnDuration"`
GroupAtType int32 `gorm:"column:group_at_type" json:"groupAtType"`
AttachedInfo string `gorm:"column:attached_info;type:varchar(1024)" json:"attachedInfo"`
Ex string `gorm:"column:ex;type:varchar(1024)" json:"ex"`
MaxSeq int64 `gorm:"column:max_seq" json:"maxSeq"`
MinSeq int64 `gorm:"column:min_seq" json:"minSeq"`
CreateTime time.Time `gorm:"column:create_time;index:create_time;autoCreateTime"`
IsMsgDestruct bool `gorm:"column:is_msg_destruct;default:false"`
MsgDestructTime int64 `gorm:"column:msg_destruct_time;default:604800"`
LatestMsgDestructTime time.Time `gorm:"column:latest_msg_destruct_time;autoCreateTime"`
}
func (ConversationModel) TableName() string {
@@ -44,26 +37,13 @@ func (ConversationModel) TableName() string {
type ConversationModelInterface interface {
Create(ctx context.Context, conversations []*ConversationModel) (err error)
Delete(ctx context.Context, groupIDs []string) (err error)
UpdateByMap(
ctx context.Context,
userIDs []string,
conversationID string,
args map[string]interface{},
) (rows int64, err error)
UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]interface{}) (rows int64, err error)
Update(ctx context.Context, conversation *ConversationModel) (err error)
Find(
ctx context.Context,
ownerUserID string,
conversationIDs []string,
) (conversations []*ConversationModel, err error)
Find(ctx context.Context, ownerUserID string, conversationIDs []string) (conversations []*ConversationModel, err error)
FindUserID(ctx context.Context, userIDs []string, conversationIDs []string) ([]string, error)
FindUserIDAllConversationID(ctx context.Context, userID string) ([]string, error)
Take(ctx context.Context, userID, conversationID string) (conversation *ConversationModel, err error)
FindConversationID(
ctx context.Context,
userID string,
conversationIDs []string,
) (existConversationID []string, err error)
FindConversationID(ctx context.Context, userID string, conversationIDs []string) (existConversationID []string, err error)
FindUserIDAllConversations(ctx context.Context, userID string) (conversations []*ConversationModel, err error)
FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error)
GetUserRecvMsgOpt(ctx context.Context, ownerUserID, conversationID string) (opt int, err error)
@@ -71,5 +51,6 @@ type ConversationModelInterface interface {
GetAllConversationIDs(ctx context.Context) ([]string, error)
GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (hashReadSeqs map[string]int64, err error)
GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*ConversationModel, error)
GetConversationIDsNeedDestruct(ctx context.Context) ([]*ConversationModel, error)
NewTx(tx any) ConversationModelInterface
}
pkg/common/db/table/relation/group.go
+4
View File
@@ -58,4 +58,8 @@ type GroupModelInterface interface {
pageNumber, showNumber int32,
) (total uint32, groups []*GroupModel, err error)
GetGroupIDsByGroupType(ctx context.Context, groupType int) (groupIDs []string, err error)
// 获取群总数
CountTotal(ctx context.Context, before *time.Time) (count int64, err error)
// 获取范围内群增量
CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
}
pkg/common/db/table/relation/object.go
+31
View File
@@ -0,0 +1,31 @@
package relation
import (
"context"
"time"
)
const (
ObjectInfoModelTableName = "object"
)
type ObjectModel struct {
Name string `gorm:"column:name;primary_key"`
UserID string `gorm:"column:user_id"`
Hash string `gorm:"column:hash"`
Key string `gorm:"column:key"`
Size int64 `gorm:"column:size"`
ContentType string `gorm:"column:content_type"`
Cause string `gorm:"column:cause"`
CreateTime time.Time `gorm:"column:create_time"`
}
func (ObjectModel) TableName() string {
return ObjectInfoModelTableName
}
type ObjectInfoModelInterface interface {
NewTx(tx any) ObjectInfoModelInterface
SetObject(ctx context.Context, obj *ObjectModel) error
Take(ctx context.Context, name string) (*ObjectModel, error)
}
pkg/common/db/table/relation/object_hash.go
-44
View File
@@ -1,44 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package relation
import (
"context"
"time"
)
const (
ObjectHashModelTableName = "object_hash"
)
type ObjectHashModel struct {
Hash string `gorm:"column:hash;primary_key;size:32"`
Engine string `gorm:"column:engine;primary_key;size:16"`
Size int64 `gorm:"column:size"`
Bucket string `gorm:"column:bucket"`
Name string `gorm:"column:name"`
CreateTime time.Time `gorm:"column:create_time"`
}
func (ObjectHashModel) TableName() string {
return ObjectHashModelTableName
}
type ObjectHashModelInterface interface {
NewTx(tx any) ObjectHashModelInterface
Take(ctx context.Context, hash string, engine string) (*ObjectHashModel, error)
Create(ctx context.Context, h []*ObjectHashModel) error
DeleteNoCitation(ctx context.Context, engine string, num int) (list []*ObjectHashModel, err error)
}
pkg/common/db/table/relation/object_info.go
-43
View File
@@ -1,43 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package relation
import (
"context"
"time"
)
const (
ObjectInfoModelTableName = "object_info"
)
type ObjectInfoModel struct {
Name string `gorm:"column:name;primary_key"`
Hash string `gorm:"column:hash"`
ContentType string `gorm:"column:content_type"`
ValidTime *time.Time `gorm:"column:valid_time"`
CreateTime time.Time `gorm:"column:create_time"`
}
func (ObjectInfoModel) TableName() string {
return ObjectInfoModelTableName
}
type ObjectInfoModelInterface interface {
NewTx(tx any) ObjectInfoModelInterface
SetObject(ctx context.Context, obj *ObjectInfoModel) error
Take(ctx context.Context, name string) (*ObjectInfoModel, error)
DeleteExpiration(ctx context.Context, expiration time.Time) error
}
pkg/common/db/table/relation/object_put.go
-51
View File
@@ -1,51 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package relation
import (
"context"
"time"
)
const (
ObjectPutModelTableName = "object_put"
)
type ObjectPutModel struct {
PutID string `gorm:"column:put_id;primary_key"`
Hash string `gorm:"column:hash"`
Path string `gorm:"column:path"`
Name string `gorm:"column:name"`
ContentType string `gorm:"column:content_type"`
ObjectSize int64 `gorm:"column:object_size"`
FragmentSize int64 `gorm:"column:fragment_size"`
PutURLsHash string `gorm:"column:put_urls_hash"`
ValidTime *time.Time `gorm:"column:valid_time"`
EffectiveTime time.Time `gorm:"column:effective_time"`
CreateTime time.Time `gorm:"column:create_time"`
}
func (ObjectPutModel) TableName() string {
return ObjectPutModelTableName
}
type ObjectPutModelInterface interface {
NewTx(tx any) ObjectPutModelInterface
Create(ctx context.Context, m []*ObjectPutModel) error
Take(ctx context.Context, putID string) (*ObjectPutModel, error)
SetCompleted(ctx context.Context, putID string) error
FindExpirationPut(ctx context.Context, expirationTime time.Time, num int) ([]*ObjectPutModel, error)
DelPut(ctx context.Context, ids []string) error
}
pkg/common/db/table/relation/user.go
+2 -2
View File
@@ -28,7 +28,7 @@ type UserModel struct {
Nickname string `gorm:"column:name;size:255"`
FaceURL string `gorm:"column:face_url;size:255"`
Ex string `gorm:"column:ex;size:1024"`
CreateTime time.Time `gorm:"column:create_time;index:create_time; autoCreateTime"`
CreateTime time.Time `gorm:"column:create_time;index:create_time;autoCreateTime"`
AppMangerLevel int32 `gorm:"column:app_manger_level;default:18"`
GlobalRecvMsgOpt int32 `gorm:"column:global_recv_msg_opt"`
}
@@ -66,7 +66,7 @@ type UserModelInterface interface {
GetAllUserID(ctx context.Context) (userIDs []string, err error)
GetUserGlobalRecvMsgOpt(ctx context.Context, userID string) (opt int, err error)
// 获取用户总数
CountTotal(ctx context.Context) (count int64, err error)
CountTotal(ctx context.Context, before *time.Time) (count int64, err error)
// 获取范围内用户增量
CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error)
}
pkg/common/db/table/unrelation/extend_msg_set.go
-125
View File
@@ -1,125 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package unrelation
import (
"context"
"strconv"
"strings"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
)
const (
CExtendMsgSet = "extend_msgs"
ExtendMsgMaxNum = 100
)
type ExtendMsgSetModel struct {
ConversationID string `bson:"source_id" json:"conversationID"`
SessionType int32 `bson:"session_type" json:"sessionType"`
ExtendMsgs map[string]ExtendMsgModel `bson:"extend_msgs" json:"extendMsgs"`
ExtendMsgNum int32 `bson:"extend_msg_num" json:"extendMsgNum"`
CreateTime int64 `bson:"create_time" json:"createTime"` // this block's create time
MaxMsgUpdateTime int64 `bson:"max_msg_update_time" json:"maxMsgUpdateTime"` // index find msg
}
type KeyValueModel struct {
TypeKey string `bson:"type_key" json:"typeKey"`
Value string `bson:"value" json:"value"`
LatestUpdateTime int64 `bson:"latest_update_time" json:"latestUpdateTime"`
}
type ExtendMsgModel struct {
ReactionExtensionList map[string]KeyValueModel `bson:"reaction_extension_list" json:"reactionExtensionList"`
ClientMsgID string `bson:"client_msg_id" json:"clientMsgID"`
MsgFirstModifyTime int64 `bson:"msg_first_modify_time" json:"msgFirstModifyTime"` // this extendMsg create time
AttachedInfo string `bson:"attached_info" json:"attachedInfo"`
Ex string `bson:"ex" json:"ex"`
}
type ExtendMsgSetModelInterface interface {
CreateExtendMsgSet(ctx context.Context, set *ExtendMsgSetModel) error
GetAllExtendMsgSet(
ctx context.Context,
conversationID string,
opts *GetAllExtendMsgSetOpts,
) (sets []*ExtendMsgSetModel, err error)
GetExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
maxMsgUpdateTime int64,
) (*ExtendMsgSetModel, error)
InsertExtendMsg(ctx context.Context, conversationID string, sessionType int32, msg *ExtendMsgModel) error
InsertOrUpdateReactionExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
msgFirstModifyTime int64,
reactionExtensionList map[string]*KeyValueModel,
) error
DeleteReactionExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
msgFirstModifyTime int64,
reactionExtensionList map[string]*KeyValueModel,
) error
TakeExtendMsg(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
maxMsgUpdateTime int64,
) (extendMsg *ExtendMsgModel, err error)
}
func (ExtendMsgSetModel) TableName() string {
return CExtendMsgSet
}
func (ExtendMsgSetModel) GetExtendMsgMaxNum() int32 {
return ExtendMsgMaxNum
}
func (ExtendMsgSetModel) GetConversationID(ID string, index int32) string {
return ID + ":" + strconv.Itoa(int(index))
}
func (e *ExtendMsgSetModel) SplitConversationIDAndGetIndex() int32 {
l := strings.Split(e.ConversationID, ":")
index, _ := strconv.Atoi(l[len(l)-1])
return int32(index)
}
type GetAllExtendMsgSetOpts struct {
ExcludeExtendMsgs bool
}
func (ExtendMsgSetModel) Pb2Model(reactionExtensionList map[string]*sdkws.KeyValue) map[string]*KeyValueModel {
r := make(map[string]*KeyValueModel)
for key, value := range reactionExtensionList {
r[key] = &KeyValueModel{
TypeKey: value.TypeKey,
Value: value.Value,
LatestUpdateTime: value.LatestUpdateTime,
}
}
return r
}
pkg/common/db/table/unrelation/msg.go
+13 -1
View File
@@ -17,6 +17,7 @@ package unrelation
import (
"context"
"strconv"
"time"
"go.mongodb.org/mongo-driver/mongo"
@@ -36,7 +37,6 @@ type MsgDocModel struct {
}
type RevokeModel struct {
ID string `bson:"id"`
Role int32 `bson:"role"`
UserID string `bson:"user_id"`
Nickname string `bson:"nickname"`
@@ -83,6 +83,16 @@ type MsgInfoModel struct {
IsRead bool `bson:"is_read"`
}
type UserCount struct {
UserID string `bson:"user_id"`
Count int64 `bson:"count"`
}
type GroupCount struct {
GroupID string `bson:"group_id"`
Count int64 `bson:"count"`
}
type MsgDocModelInterface interface {
PushMsgsToDoc(ctx context.Context, docID string, msgsToMongo []MsgInfoModel) error
Create(ctx context.Context, model *MsgDocModel) error
@@ -98,6 +108,8 @@ type MsgDocModelInterface interface {
GetMsgDocModelByIndex(ctx context.Context, conversationID string, index, sort int64) (*MsgDocModel, error)
DeleteMsgsInOneDocByIndex(ctx context.Context, docID string, indexes []int) error
MarkSingleChatMsgsAsRead(ctx context.Context, userID string, docID string, indexes []int64) error
RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*UserCount, dateCount map[string]int64, err error)
RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*GroupCount, dateCount map[string]int64, err error)
}
func (MsgDocModel) TableName() string {
pkg/common/db/unrelation/extend_msg.go
-230
View File
@@ -1,230 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package unrelation
import (
"context"
"errors"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
unRelationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
)
type ExtendMsgSetMongoDriver struct {
mgoDB *mongo.Database
ExtendMsgSetCollection *mongo.Collection
}
func NewExtendMsgSetMongoDriver(mgoDB *mongo.Database) unRelationTb.ExtendMsgSetModelInterface {
return &ExtendMsgSetMongoDriver{mgoDB: mgoDB, ExtendMsgSetCollection: mgoDB.Collection(unRelationTb.CExtendMsgSet)}
}
func (e *ExtendMsgSetMongoDriver) CreateExtendMsgSet(ctx context.Context, set *unRelationTb.ExtendMsgSetModel) error {
_, err := e.ExtendMsgSetCollection.InsertOne(ctx, set)
return err
}
func (e *ExtendMsgSetMongoDriver) GetAllExtendMsgSet(
ctx context.Context,
ID string,
opts *unRelationTb.GetAllExtendMsgSetOpts,
) (sets []*unRelationTb.ExtendMsgSetModel, err error) {
regex := fmt.Sprintf("^%s", ID)
var findOpts *options.FindOptions
if opts != nil {
if opts.ExcludeExtendMsgs {
findOpts = &options.FindOptions{}
findOpts.SetProjection(bson.M{"extend_msgs": 0})
}
}
cursor, err := e.ExtendMsgSetCollection.Find(ctx, bson.M{"doc_id": primitive.Regex{Pattern: regex}}, findOpts)
if err != nil {
return nil, utils.Wrap(err, "")
}
err = cursor.All(ctx, &sets)
if err != nil {
return nil, utils.Wrap(err, fmt.Sprintf("cursor is %s", cursor.Current.String()))
}
return sets, nil
}
func (e *ExtendMsgSetMongoDriver) GetExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
maxMsgUpdateTime int64,
) (*unRelationTb.ExtendMsgSetModel, error) {
var err error
findOpts := options.Find().
SetLimit(1).
SetSkip(0).
SetSort(bson.M{"source_id": -1}).
SetProjection(bson.M{"extend_msgs": 0})
// update newest
find := bson.M{
"source_id": primitive.Regex{Pattern: fmt.Sprintf("^%s", conversationID)},
"session_type": sessionType,
}
if maxMsgUpdateTime > 0 {
find["max_msg_update_time"] = maxMsgUpdateTime
}
result, err := e.ExtendMsgSetCollection.Find(ctx, find, findOpts)
if err != nil {
return nil, utils.Wrap(err, "")
}
var setList []unRelationTb.ExtendMsgSetModel
if err := result.All(ctx, &setList); err != nil {
return nil, utils.Wrap(err, "")
}
if len(setList) == 0 {
return nil, nil
}
return &setList[0], nil
}
// InsertExtendMsg first modify msg.
func (e *ExtendMsgSetMongoDriver) InsertExtendMsg(
ctx context.Context,
conversationID string,
sessionType int32,
msg *unRelationTb.ExtendMsgModel,
) error {
set, err := e.GetExtendMsgSet(ctx, conversationID, sessionType, 0)
if err != nil {
return utils.Wrap(err, "")
}
if set == nil || set.ExtendMsgNum >= set.GetExtendMsgMaxNum() {
var index int32
if set != nil {
index = set.SplitConversationIDAndGetIndex()
}
err = e.CreateExtendMsgSet(ctx, &unRelationTb.ExtendMsgSetModel{
ConversationID: set.GetConversationID(conversationID, index),
SessionType: sessionType,
ExtendMsgs: map[string]unRelationTb.ExtendMsgModel{msg.ClientMsgID: *msg},
ExtendMsgNum: 1,
CreateTime: msg.MsgFirstModifyTime,
MaxMsgUpdateTime: msg.MsgFirstModifyTime,
})
} else {
_, err = e.ExtendMsgSetCollection.UpdateOne(ctx, bson.M{"conversation_id": set.ConversationID, "session_type": sessionType}, bson.M{"$set": bson.M{"max_msg_update_time": msg.MsgFirstModifyTime, "$inc": bson.M{"extend_msg_num": 1}, fmt.Sprintf("extend_msgs.%s", msg.ClientMsgID): msg}})
}
return utils.Wrap(err, "")
}
// InsertOrUpdateReactionExtendMsgSet insert or update.
func (e *ExtendMsgSetMongoDriver) InsertOrUpdateReactionExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
msgFirstModifyTime int64,
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
) error {
var updateBson = bson.M{}
for _, v := range reactionExtensionList {
updateBson[fmt.Sprintf("extend_msgs.%s.%s", clientMsgID, v.TypeKey)] = v
}
upsert := true
opt := &options.UpdateOptions{
Upsert: &upsert,
}
set, err := e.GetExtendMsgSet(ctx, conversationID, sessionType, msgFirstModifyTime)
if err != nil {
return utils.Wrap(err, "")
}
if set == nil {
return errors.New(fmt.Sprintf("conversationID %s has no set", conversationID))
}
_, err = e.ExtendMsgSetCollection.UpdateOne(
ctx,
bson.M{"source_id": set.ConversationID, "session_type": sessionType},
bson.M{"$set": updateBson},
opt,
)
return utils.Wrap(err, "")
}
// DeleteReactionExtendMsgSet delete TypeKey.
func (e *ExtendMsgSetMongoDriver) DeleteReactionExtendMsgSet(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
msgFirstModifyTime int64,
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
) error {
var updateBson = bson.M{}
for _, v := range reactionExtensionList {
updateBson[fmt.Sprintf("extend_msgs.%s.%s", clientMsgID, v.TypeKey)] = ""
}
set, err := e.GetExtendMsgSet(ctx, conversationID, sessionType, msgFirstModifyTime)
if err != nil {
return utils.Wrap(err, "")
}
if set == nil {
return errors.New(fmt.Sprintf("conversationID %s has no set", conversationID))
}
_, err = e.ExtendMsgSetCollection.UpdateOne(
ctx,
bson.M{"source_id": set.ConversationID, "session_type": sessionType},
bson.M{"$unset": updateBson},
)
return err
}
func (e *ExtendMsgSetMongoDriver) TakeExtendMsg(
ctx context.Context,
conversationID string,
sessionType int32,
clientMsgID string,
maxMsgUpdateTime int64,
) (extendMsg *unRelationTb.ExtendMsgModel, err error) {
findOpts := options.Find().
SetLimit(1).
SetSkip(0).
SetSort(bson.M{"source_id": -1}).
SetProjection(bson.M{fmt.Sprintf("extend_msgs.%s", clientMsgID): 1})
regex := fmt.Sprintf("^%s", conversationID)
result, err := e.ExtendMsgSetCollection.Find(
ctx,
bson.M{
"source_id": primitive.Regex{Pattern: regex},
"session_type": sessionType,
"max_msg_update_time": bson.M{"$lte": maxMsgUpdateTime},
},
findOpts,
)
if err != nil {
return nil, utils.Wrap(err, "")
}
var setList []unRelationTb.ExtendMsgSetModel
if err := result.All(ctx, &setList); err != nil {
return nil, utils.Wrap(err, "")
}
if len(setList) == 0 {
return nil, utils.Wrap(errors.New("GetExtendMsg failed, len(setList) == 0"), "")
}
if v, ok := setList[0].ExtendMsgs[clientMsgID]; ok {
return &v, nil
}
return nil, errors.New(fmt.Sprintf("cant find client msg id: %s", clientMsgID))
}
pkg/common/db/unrelation/mongo.go
-4
View File
@@ -107,10 +107,6 @@ func (m *Mongo) CreateSuperGroupIndex() error {
return nil
}
func (m *Mongo) CreateExtendMsgSetIndex() error {
return m.createMongoIndex(unrelation.CExtendMsgSet, true, "-create_time", "work_moment_id")
}
func (m *Mongo) createMongoIndex(collection string, isUnique bool, keys ...string) error {
db := m.db.Database(config.Config.Mongo.Database).Collection(collection)
opts := options.CreateIndexes().SetMaxTime(10 * time.Second)
pkg/common/db/unrelation/msg.go
+682 -1
View File
@@ -19,6 +19,7 @@ import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
@@ -302,7 +303,7 @@ func (m *MsgMongoDriver) GetMsgBySeqIndexIn1Doc(
}
if msg.Revoke != nil {
revokeContent := sdkws.MessageRevokedContent{
RevokerID: msg.Revoke.ID,
RevokerID: msg.Revoke.UserID,
RevokerRole: msg.Revoke.Role,
ClientMsgID: msg.Msg.ClientMsgID,
RevokerNickname: msg.Revoke.Nickname,
@@ -368,3 +369,683 @@ func (m *MsgMongoDriver) MarkSingleChatMsgsAsRead(
_, err := m.MsgCollection.BulkWrite(ctx, updates)
return err
}
// RangeUserSendCount
// db.msg.aggregate([
//
// {
// $match: {
// "msgs.msg.send_time": {
// "$gte": 0,
// "$lt": 1788122092317
// }
// }
// },
// {
// "$addFields": {
// "msgs": {
// "$filter": {
// "input": "$msgs",
// "as": "item",
// "cond": {
// "$and": [
// {
// $gte: ["$$item.msg.send_time", 0]
// },
// {
// $lt: ["$$item.msg.send_time", 1788122092317]
// }
// ]
// }
// }
// }
// }
// },
// {
// "$project": {
// "_id": 0,
//
// },
//
// },
// {
// "$project": {
// "result": {
// "$map": {
// "input": "$msgs",
// "as": "item",
// "in": {
// user_id: "$$item.msg.send_id",
// send_date: {
// $dateToString: {
// format: "%Y-%m-%d",
// date: {
// $toDate: "$$item.msg.send_time"
// }
// }
// }
// }
// }
// }
// },
//
// },
// {
// "$unwind": "$result"
// },
// {
// "$group": {
// _id: "$result.send_date",
// count: {
// $sum: 1
// },
// original: {
// $push: "$$ROOT"
// }
// }
// },
// {
// "$addFields": {
// "dates": "$$ROOT"
// }
// },
// {
// "$project": {
// "_id": 0,
// "count": 0,
// "dates.original": 0,
//
// },
//
// },
// {
// "$group": {
// _id: null,
// count: {
// $sum: 1
// },
// dates: {
// $push: "$dates"
// },
// original: {
// $push: "$original"
// },
//
// }
// },
// {
// "$unwind": "$original"
// },
// {
// "$unwind": "$original"
// },
// {
// "$group": {
// _id: "$original.result.user_id",
// count: {
// $sum: 1
// },
// original: {
// $push: "$dates"
// },
//
// }
// },
// {
// "$addFields": {
// "dates": {
// $arrayElemAt: ["$original", 0]
// }
// }
// },
// {
// "$project": {
// original: 0
// }
// },
// {
// $sort: {
// count: - 1
// }
// },
// {
// "$group": {
// _id: null,
// user_count: {
// $sum: 1
// },
// users: {
// $push: "$$ROOT"
// },
//
// }
// },
// {
// "$addFields": {
// "dates": {
// $arrayElemAt: ["$users", 0]
// }
// }
// },
// {
// "$addFields": {
// "dates": "$dates.dates"
// }
// },
// {
// "$project": {
// _id: 0,
// "users.dates": 0,
//
// }
// },
// {
// "$addFields": {
// "msg_count": {
// $sum: "$users.count"
// }
// }
// },
// {
// "$addFields": {
// users: {
// $slice: ["$users", 0, 10]
// }
// }
// }
//
// ])
func (m *MsgMongoDriver) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*table.UserCount, dateCount map[string]int64, err error) {
var sort int
if ase {
sort = 1
} else {
sort = -1
}
type Result struct {
MsgCount int64 `bson:"msg_count"`
UserCount int64 `bson:"user_count"`
Users []struct {
UserID string `bson:"_id"`
Count int64 `bson:"count"`
} `bson:"users"`
Dates []struct {
Date string `bson:"_id"`
Count int64 `bson:"count"`
} `bson:"dates"`
}
or := bson.A{
bson.M{
"doc_id": bson.M{
"$regex": "^si_",
"$options": "i",
},
},
}
if group {
or = append(or,
bson.M{
"doc_id": bson.M{
"$regex": "^g_",
"$options": "i",
},
},
bson.M{
"doc_id": bson.M{
"$regex": "^sg_",
"$options": "i",
},
},
)
}
pipeline := bson.A{
bson.M{
"$match": bson.M{
"$and": bson.A{
bson.M{
"msgs.msg.send_time": bson.M{
"$gte": start.UnixMilli(),
"$lt": end.UnixMilli(),
},
},
bson.M{
"$or": or,
},
},
},
},
bson.M{
"$addFields": bson.M{
"msgs": bson.M{
"$filter": bson.M{
"input": "$msgs",
"as": "item",
"cond": bson.M{
"$and": bson.A{
bson.M{
"$gte": bson.A{
"$$item.msg.send_time", start.UnixMilli(),
},
},
bson.M{
"$lt": bson.A{
"$$item.msg.send_time", end.UnixMilli(),
},
},
},
},
},
},
},
},
bson.M{
"$project": bson.M{
"_id": 0,
},
},
bson.M{
"$project": bson.M{
"result": bson.M{
"$map": bson.M{
"input": "$msgs",
"as": "item",
"in": bson.M{
"user_id": "$$item.msg.send_id",
"send_date": bson.M{
"$dateToString": bson.M{
"format": "%Y-%m-%d",
"date": bson.M{
"$toDate": "$$item.msg.send_time", // 毫秒时间戳
},
},
},
},
},
},
},
},
bson.M{
"$unwind": "$result",
},
bson.M{
"$group": bson.M{
"_id": "$result.send_date",
"count": bson.M{
"$sum": 1,
},
"original": bson.M{
"$push": "$$ROOT",
},
},
},
bson.M{
"$addFields": bson.M{
"dates": "$$ROOT",
},
},
bson.M{
"$project": bson.M{
"_id": 0,
"count": 0,
"dates.original": 0,
},
},
bson.M{
"$group": bson.M{
"_id": nil,
"count": bson.M{
"$sum": 1,
},
"dates": bson.M{
"$push": "$dates",
},
"original": bson.M{
"$push": "$original",
},
},
},
bson.M{
"$unwind": "$original",
},
bson.M{
"$unwind": "$original",
},
bson.M{
"$group": bson.M{
"_id": "$original.result.user_id",
"count": bson.M{
"$sum": 1,
},
"original": bson.M{
"$push": "$dates",
},
},
},
bson.M{
"$addFields": bson.M{
"dates": bson.M{
"$arrayElemAt": bson.A{"$original", 0},
},
},
},
bson.M{
"$project": bson.M{
"original": 0,
},
},
bson.M{
"$sort": bson.M{
"count": sort,
},
},
bson.M{
"$group": bson.M{
"_id": nil,
"user_count": bson.M{
"$sum": 1,
},
"users": bson.M{
"$push": "$$ROOT",
},
},
},
bson.M{
"$addFields": bson.M{
"dates": bson.M{
"$arrayElemAt": bson.A{"$users", 0},
},
},
},
bson.M{
"$addFields": bson.M{
"dates": "$dates.dates",
},
},
bson.M{
"$project": bson.M{
"_id": 0,
"users.dates": 0,
},
},
bson.M{
"$addFields": bson.M{
"msg_count": bson.M{
"$sum": "$users.count",
},
},
},
bson.M{
"$addFields": bson.M{
"users": bson.M{
"$slice": bson.A{"$users", pageNumber - 1, showNumber},
},
},
},
}
cur, err := m.MsgCollection.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(true))
if err != nil {
return 0, 0, nil, nil, errs.Wrap(err)
}
defer cur.Close(ctx)
var result []Result
if err := cur.All(ctx, &result); err != nil {
return 0, 0, nil, nil, errs.Wrap(err)
}
if len(result) == 0 {
return 0, 0, nil, nil, errs.Wrap(err)
}
users = make([]*table.UserCount, len(result[0].Users))
for i, r := range result[0].Users {
users[i] = &table.UserCount{
UserID: r.UserID,
Count: r.Count,
}
}
dateCount = make(map[string]int64)
for _, r := range result[0].Dates {
dateCount[r.Date] = r.Count
}
return result[0].MsgCount, result[0].UserCount, users, dateCount, nil
}
func (m *MsgMongoDriver) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*table.GroupCount, dateCount map[string]int64, err error) {
var sort int
if ase {
sort = 1
} else {
sort = -1
}
type Result struct {
MsgCount int64 `bson:"msg_count"`
UserCount int64 `bson:"user_count"`
Groups []struct {
GroupID string `bson:"_id"`
Count int64 `bson:"count"`
} `bson:"groups"`
Dates []struct {
Date string `bson:"_id"`
Count int64 `bson:"count"`
} `bson:"dates"`
}
pipeline := bson.A{
bson.M{
"$match": bson.M{
"$and": bson.A{
bson.M{
"msgs.msg.send_time": bson.M{
"$gte": start.UnixMilli(),
"$lt": end.UnixMilli(),
},
},
bson.M{
"$or": bson.A{
bson.M{
"doc_id": bson.M{
"$regex": "^g_",
"$options": "i",
},
},
bson.M{
"doc_id": bson.M{
"$regex": "^sg_",
"$options": "i",
},
},
},
},
},
},
},
bson.M{
"$addFields": bson.M{
"msgs": bson.M{
"$filter": bson.M{
"input": "$msgs",
"as": "item",
"cond": bson.M{
"$and": bson.A{
bson.M{
"$gte": bson.A{
"$$item.msg.send_time", start.UnixMilli(),
},
},
bson.M{
"$lt": bson.A{
"$$item.msg.send_time", end.UnixMilli(),
},
},
},
},
},
},
},
},
bson.M{
"$project": bson.M{
"_id": 0,
},
},
bson.M{
"$project": bson.M{
"result": bson.M{
"$map": bson.M{
"input": "$msgs",
"as": "item",
"in": bson.M{
"group_id": "$$item.msg.group_id",
"send_date": bson.M{
"$dateToString": bson.M{
"format": "%Y-%m-%d",
"date": bson.M{
"$toDate": "$$item.msg.send_time", // 毫秒时间戳
},
},
},
},
},
},
},
},
bson.M{
"$unwind": "$result",
},
bson.M{
"$group": bson.M{
"_id": "$result.send_date",
"count": bson.M{
"$sum": 1,
},
"original": bson.M{
"$push": "$$ROOT",
},
},
},
bson.M{
"$addFields": bson.M{
"dates": "$$ROOT",
},
},
bson.M{
"$project": bson.M{
"_id": 0,
"count": 0,
"dates.original": 0,
},
},
bson.M{
"$group": bson.M{
"_id": nil,
"count": bson.M{
"$sum": 1,
},
"dates": bson.M{
"$push": "$dates",
},
"original": bson.M{
"$push": "$original",
},
},
},
bson.M{
"$unwind": "$original",
},
bson.M{
"$unwind": "$original",
},
bson.M{
"$group": bson.M{
"_id": "$original.result.group_id",
"count": bson.M{
"$sum": 1,
},
"original": bson.M{
"$push": "$dates",
},
},
},
bson.M{
"$addFields": bson.M{
"dates": bson.M{
"$arrayElemAt": bson.A{"$original", 0},
},
},
},
bson.M{
"$project": bson.M{
"original": 0,
},
},
bson.M{
"$sort": bson.M{
"count": sort,
},
},
bson.M{
"$group": bson.M{
"_id": nil,
"user_count": bson.M{
"$sum": 1,
},
"groups": bson.M{
"$push": "$$ROOT",
},
},
},
bson.M{
"$addFields": bson.M{
"dates": bson.M{
"$arrayElemAt": bson.A{"$groups", 0},
},
},
},
bson.M{
"$addFields": bson.M{
"dates": "$dates.dates",
},
},
bson.M{
"$project": bson.M{
"_id": 0,
"groups.dates": 0,
},
},
bson.M{
"$addFields": bson.M{
"msg_count": bson.M{
"$sum": "$groups.count",
},
},
},
bson.M{
"$addFields": bson.M{
"groups": bson.M{
"$slice": bson.A{"$groups", pageNumber - 1, showNumber},
},
},
},
}
cur, err := m.MsgCollection.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(true))
if err != nil {
return 0, 0, nil, nil, errs.Wrap(err)
}
defer cur.Close(ctx)
var result []Result
if err := cur.All(ctx, &result); err != nil {
return 0, 0, nil, nil, errs.Wrap(err)
}
if len(result) == 0 {
return 0, 0, nil, nil, errs.Wrap(err)
}
groups = make([]*table.GroupCount, len(result[0].Groups))
for i, r := range result[0].Groups {
groups[i] = &table.GroupCount{
GroupID: r.GroupID,
Count: r.Count,
}
}
dateCount = make(map[string]int64)
for _, r := range result[0].Dates {
dateCount[r.Date] = r.Count
}
return result[0].MsgCount, result[0].UserCount, groups, dateCount, nil
}
pkg/common/kafka/consumer_group.go
+14
View File
@@ -1,3 +1,17 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kafka
import (
pkg/common/log/sql_logger.go
+5 -55
View File
@@ -1,17 +1,3 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package log
import (
@@ -31,11 +17,7 @@ type SqlLogger struct {
SlowThreshold time.Duration
}
func NewSqlLogger(
logLevel gormLogger.LogLevel,
ignoreRecordNotFoundError bool,
slowThreshold time.Duration,
) *SqlLogger {
func NewSqlLogger(logLevel gormLogger.LogLevel, ignoreRecordNotFoundError bool, slowThreshold time.Duration) *SqlLogger {
return &SqlLogger{
LogLevel: logLevel,
IgnoreRecordNotFoundError: ignoreRecordNotFoundError,
@@ -70,17 +52,7 @@ func (l *SqlLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql s
case err != nil && l.LogLevel >= gormLogger.Error && (!errors.Is(err, gorm.ErrRecordNotFound) || !l.IgnoreRecordNotFoundError):
sql, rows := fc()
if rows == -1 {
ZError(
ctx,
"sql exec detail",
err,
"gorm",
gormUtils.FileWithLineNum(),
"elapsed time",
fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6),
"sql",
sql,
)
ZError(ctx, "sql exec detail", err, "gorm", gormUtils.FileWithLineNum(), "elapsed time", fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6), "sql", sql)
} else {
ZError(ctx, "sql exec detail", err, "gorm", gormUtils.FileWithLineNum(), "elapsed time", fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6), "rows", rows, "sql", sql)
}
@@ -88,36 +60,14 @@ func (l *SqlLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql s
sql, rows := fc()
slowLog := fmt.Sprintf("SLOW SQL >= %v", l.SlowThreshold)
if rows == -1 {
ZWarn(
ctx,
"sql exec detail",
nil,
"gorm",
gormUtils.FileWithLineNum(),
nil,
"slow sql",
slowLog,
"elapsed time",
fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6),
"sql",
sql,
)
ZWarn(ctx, "sql exec detail", nil, "gorm", gormUtils.FileWithLineNum(), "slow sql", slowLog, "elapsed time", fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6), "sql", sql)
} else {
ZWarn(ctx, "sql exec detail", nil, "gorm", gormUtils.FileWithLineNum(), nil, "slow sql", slowLog, "elapsed time", fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6), "rows", rows, "sql", sql)
ZWarn(ctx, "sql exec detail", nil, "gorm", gormUtils.FileWithLineNum(), "slow sql", slowLog, "elapsed time", fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6), "rows", rows, "sql", sql)
}
case l.LogLevel == gormLogger.Info:
sql, rows := fc()
if rows == -1 {
ZDebug(
ctx,
"sql exec detail",
"gorm",
gormUtils.FileWithLineNum(),
"elapsed time",
fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6),
"sql",
sql,
)
ZDebug(ctx, "sql exec detail", "gorm", gormUtils.FileWithLineNum(), "elapsed time", fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6), "sql", sql)
} else {
ZDebug(ctx, "sql exec detail", "gorm", gormUtils.FileWithLineNum(), "elapsed time", fmt.Sprintf("%f(ms)", float64(elapsed.Nanoseconds())/1e6), "rows", rows, "sql", sql)
}
pkg/common/mw/rpc_client_interceptor.go
+1 -1
View File
@@ -31,7 +31,7 @@ import (
)
func GrpcClient() grpc.DialOption {
return grpc.WithUnaryInterceptor(RpcClientInterceptor)
return grpc.WithChainUnaryInterceptor(RpcClientInterceptor)
}
func RpcClientInterceptor(
pkg/common/mw/rpc_server_interceptor.go
+3 -3
View File
@@ -52,7 +52,6 @@ func RpcServerInterceptor(
handler grpc.UnaryHandler,
) (resp interface{}, err error) {
log.ZDebug(ctx, "rpc server req", "req", rpcString(req))
//defer func() {
// if r := recover(); r != nil {
// log.ZError(ctx, "rpc panic", nil, "FullMethod", info.FullMethod, "type:", fmt.Sprintf("%T", r), "panic:", r)
@@ -177,12 +176,13 @@ func RpcServerInterceptor(
}
details, err := grpcStatus.WithDetails(errInfo)
if err != nil {
panic(err)
log.ZWarn(ctx, "rpc server resp WithDetails error", err, "funcName", funcName)
return nil, errs.Wrap(err)
}
log.ZWarn(ctx, "rpc server resp", err, "funcName", funcName)
return nil, details.Err()
}
func GrpcServer() grpc.ServerOption {
return grpc.UnaryInterceptor(RpcServerInterceptor)
return grpc.ChainUnaryInterceptor(RpcServerInterceptor)
}
pkg/common/tokenverify/jwt_token.go
+2 -2
View File
@@ -84,7 +84,7 @@ func CheckAccessV3(ctx context.Context, ownerUserID string) (err error) {
if opUserID == ownerUserID {
return nil
}
return errs.ErrIdentity.Wrap(utils.GetSelfFuncName())
return errs.ErrNoPermission.Wrap(utils.GetSelfFuncName())
}
func IsAppManagerUid(ctx context.Context) bool {
@@ -95,7 +95,7 @@ func CheckAdmin(ctx context.Context) error {
if utils.IsContain(mcontext.GetOpUserID(ctx), config.Config.Manager.UserID) {
return nil
}
return errs.ErrIdentity.Wrap(fmt.Sprintf("user %s is not admin userID", mcontext.GetOpUserID(ctx)))
return errs.ErrNoPermission.Wrap(fmt.Sprintf("user %s is not admin userID", mcontext.GetOpUserID(ctx)))
}
func ParseRedisInterfaceToken(redisToken interface{}) (*Claims, error) {