feat: Remove MySQL and implement it all with Mongo (#1508)

* fix: GetUserReqApplicationList error when there is a disbanded group chat

* fix: error when querying some information about disbanded group

* fix: GetUserReqApplicationList dismissed group error

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* fix: the original message referenced by the pull message processing is withdrawn

* merge

* cicd: robot automated Change

* sdkws.MsgData

* user

* interface{} -> any

* user

* third

* group

* group

* group

* group

* group

* group

* conversation

* standalone mysql db model

* tx

* s3

* group

* mongo

* group

* group

* group

* group

* group

* group

* refactor: add openim mysql to mongo refactor

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>

* refactor: add openim mysql to mongo refactor

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>

* remove mysql

* remove mysql

* friend

* friend

* friend

* friend

* friend

* friend

* group

* convert

* index

* index

* all

* all

* mysql2mongo

* data conversion

* up35

* up35

* feat: add format set

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>

* fix: fix scripts

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>

* merge main

* merge main

* Update init-config.sh

* fix: user args check

---------

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>
Co-authored-by: withchao <withchao@users.noreply.github.com>
Co-authored-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>
Co-authored-by: Xinwei Xiong <3293172751@qq.com>
This commit is contained in:
chao
2023-12-05 20:53:02 +08:00
committed by GitHub
parent 1c1322e3d2
commit c0194f6ef4
141 changed files with 4273 additions and 4858 deletions
+1 -1
View File
@@ -150,7 +150,7 @@ func (m *MessageApi) DeleteMsgPhysical(c *gin.Context) {
}
func (m *MessageApi) getSendMsgReq(c *gin.Context, req apistruct.SendMsg) (sendMsgReq *msg.SendMsgReq, err error) {
var data interface{}
var data any
log.ZDebug(c, "getSendMsgReq", "req", req.Content)
switch req.ContentType {
case constant.Text:
+4 -4
View File
@@ -22,8 +22,8 @@ import (
)
type Encoder interface {
Encode(data interface{}) ([]byte, error)
Decode(encodeData []byte, decodeData interface{}) error
Encode(data any) ([]byte, error)
Decode(encodeData []byte, decodeData any) error
}
type GobEncoder struct{}
@@ -32,7 +32,7 @@ func NewGobEncoder() *GobEncoder {
return &GobEncoder{}
}
func (g *GobEncoder) Encode(data interface{}) ([]byte, error) {
func (g *GobEncoder) Encode(data any) ([]byte, error) {
buff := bytes.Buffer{}
enc := gob.NewEncoder(&buff)
err := enc.Encode(data)
@@ -42,7 +42,7 @@ func (g *GobEncoder) Encode(data interface{}) ([]byte, error) {
return buff.Bytes(), nil
}
func (g *GobEncoder) Decode(encodeData []byte, decodeData interface{}) error {
func (g *GobEncoder) Decode(encodeData []byte, decodeData any) error {
buff := bytes.NewBuffer(encodeData)
dec := gob.NewDecoder(buff)
err := dec.Decode(decodeData)
+9 -3
View File
@@ -49,7 +49,7 @@ type LongConnServer interface {
wsHandler(w http.ResponseWriter, r *http.Request)
GetUserAllCons(userID string) ([]*Client, bool)
GetUserPlatformCons(userID string, platform int) ([]*Client, bool, bool)
Validate(s interface{}) error
Validate(s any) error
SetCacheHandler(cache cache.MsgModel)
SetDiscoveryRegistry(client discoveryregistry.SvcDiscoveryRegistry)
KickUserConn(client *Client) error
@@ -60,6 +60,12 @@ type LongConnServer interface {
MessageHandler
}
var bufferPool = sync.Pool{
New: func() any {
return make([]byte, 1024)
},
}
type WsServer struct {
port int
wsMaxConnNum int64
@@ -120,7 +126,7 @@ func (ws *WsServer) UnRegister(c *Client) {
ws.unregisterChan <- c
}
func (ws *WsServer) Validate(s interface{}) error {
func (ws *WsServer) Validate(s any) error {
//?question?
return nil
}
@@ -145,7 +151,7 @@ func NewWsServer(opts ...Option) (*WsServer, error) {
writeBufferSize: config.writeBufferSize,
handshakeTimeout: config.handshakeTimeout,
clientPool: sync.Pool{
New: func() interface{} {
New: func() any {
return new(Client)
},
},
+7 -21
View File
@@ -17,21 +17,20 @@ package msgtransfer
import (
"errors"
"fmt"
"log"
"net/http"
"sync"
"github.com/OpenIMSDK/tools/mw"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"log"
"net/http"
"sync"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/controller"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/relation"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister"
"github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics"
@@ -39,20 +38,12 @@ import (
)
type MsgTransfer struct {
persistentCH *PersistentConsumerHandler // 聊天记录持久化到mysql的消费者 订阅的topic: ws2ms_chat
historyCH *OnlineHistoryRedisConsumerHandler // 这个消费者聚合消息, 订阅的topic:ws2ms_chat, 修改通知发往msg_to_modify topic, 消息存入redis后Incr Redis, 再发消息到ms2pschat topic推送, 发消息到msg_to_mongo topic持久化
historyMongoCH *OnlineHistoryMongoConsumerHandler // mongoDB批量插入, 成功后删除redis中消息,以及处理删除通知消息删除的 订阅的topic: msg_to_mongo
// modifyCH *ModifyMsgConsumerHandler // 负责消费修改消息通知的consumer, 订阅的topic: msg_to_modify
}
func StartTransfer(prometheusPort int) error {
db, err := relation.NewGormDB()
if err != nil {
return err
}
if err := db.AutoMigrate(&relationtb.ChatLogModel{}); err != nil {
fmt.Printf("gorm: AutoMigrate ChatLogModel err: %v\n", err)
}
rdb, err := cache.NewRedis()
if err != nil {
return err
@@ -78,21 +69,16 @@ func StartTransfer(prometheusPort int) error {
client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()))
msgModel := cache.NewMsgCacheModel(rdb)
msgDocModel := unrelation.NewMsgMongoDriver(mongo.GetDatabase())
msgMysModel := relation.NewChatLogGorm(db)
chatLogDatabase := controller.NewChatLogDatabase(msgMysModel)
msgDatabase := controller.NewCommonMsgDatabase(msgDocModel, msgModel)
conversationRpcClient := rpcclient.NewConversationRpcClient(client)
groupRpcClient := rpcclient.NewGroupRpcClient(client)
msgTransfer := NewMsgTransfer(chatLogDatabase, msgDatabase, &conversationRpcClient, &groupRpcClient)
msgTransfer := NewMsgTransfer(msgDatabase, &conversationRpcClient, &groupRpcClient)
return msgTransfer.Start(prometheusPort)
}
func NewMsgTransfer(chatLogDatabase controller.ChatLogDatabase,
msgDatabase controller.CommonMsgDatabase,
conversationRpcClient *rpcclient.ConversationRpcClient, groupRpcClient *rpcclient.GroupRpcClient,
) *MsgTransfer {
func NewMsgTransfer(msgDatabase controller.CommonMsgDatabase, conversationRpcClient *rpcclient.ConversationRpcClient, groupRpcClient *rpcclient.GroupRpcClient) *MsgTransfer {
return &MsgTransfer{
persistentCH: NewPersistentConsumerHandler(chatLogDatabase), historyCH: NewOnlineHistoryRedisConsumerHandler(msgDatabase, conversationRpcClient, groupRpcClient),
historyCH: NewOnlineHistoryRedisConsumerHandler(msgDatabase, conversationRpcClient, groupRpcClient),
historyMongoCH: NewOnlineHistoryMongoConsumerHandler(msgDatabase),
}
}
@@ -61,7 +61,7 @@ type TriggerChannelValue struct {
type Cmd2Value struct {
Cmd int
Value interface{}
Value any
}
type ContextMsg struct {
message *sdkws.MsgData
@@ -1,119 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package msgtransfer
import (
"context"
"github.com/OpenIMSDK/protocol/constant"
pbmsg "github.com/OpenIMSDK/protocol/msg"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/controller"
kfk "github.com/openimsdk/open-im-server/v3/pkg/common/kafka"
"github.com/IBM/sarama"
"google.golang.org/protobuf/proto"
)
type PersistentConsumerHandler struct {
persistentConsumerGroup *kfk.MConsumerGroup
chatLogDatabase controller.ChatLogDatabase
}
func NewPersistentConsumerHandler(database controller.ChatLogDatabase) *PersistentConsumerHandler {
return &PersistentConsumerHandler{
persistentConsumerGroup: kfk.NewMConsumerGroup(&kfk.MConsumerGroupConfig{
KafkaVersion: sarama.V2_0_0_0,
OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false,
}, []string{config.Config.Kafka.LatestMsgToRedis.Topic},
config.Config.Kafka.Addr, config.Config.Kafka.ConsumerGroupID.MsgToMySql),
chatLogDatabase: database,
}
}
func (pc *PersistentConsumerHandler) handleChatWs2Mysql(
ctx context.Context,
cMsg *sarama.ConsumerMessage,
msgKey string,
_ sarama.ConsumerGroupSession,
) {
msg := cMsg.Value
var tag bool
msgFromMQ := pbmsg.MsgDataToMQ{}
err := proto.Unmarshal(msg, &msgFromMQ)
if err != nil {
log.ZError(ctx, "msg_transfer Unmarshal msg err", err)
return
}
log.ZDebug(ctx, "handleChatWs2Mysql", "msg", msgFromMQ.MsgData)
// Control whether to store history messages (mysql)
isPersist := utils.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsPersistent)
// Only process receiver data
if isPersist {
switch msgFromMQ.MsgData.SessionType {
case constant.SingleChatType, constant.NotificationChatType:
if msgKey == msgFromMQ.MsgData.RecvID {
tag = true
}
case constant.GroupChatType:
if msgKey == msgFromMQ.MsgData.SendID {
tag = true
}
case constant.SuperGroupChatType:
tag = true
}
if tag {
log.ZInfo(ctx, "msg_transfer msg persisting", "msg", string(msg))
if err = pc.chatLogDatabase.CreateChatLog(&msgFromMQ); err != nil {
log.ZError(ctx, "Message insert failed", err, "msg", msgFromMQ.String())
return
}
}
}
}
func (PersistentConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil }
func (PersistentConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil }
func (pc *PersistentConsumerHandler) ConsumeClaim(
sess sarama.ConsumerGroupSession,
claim sarama.ConsumerGroupClaim,
) error {
for msg := range claim.Messages() {
ctx := pc.persistentConsumerGroup.GetContextFromMsg(msg)
log.ZDebug(
ctx,
"kafka get info to mysql",
"msgTopic",
msg.Topic,
"msgPartition",
msg.Partition,
"msg",
string(msg.Value),
"key",
string(msg.Key),
)
if len(msg.Value) != 0 {
pc.handleChatWs2Mysql(ctx, msg, string(msg.Key), sess)
} else {
log.ZError(ctx, "msg get from kafka but is nil", nil, "key", msg.Key)
}
sess.MarkMessage(msg, "")
}
return nil
}
+3 -3
View File
@@ -21,9 +21,9 @@ import (
)
type Resp struct {
Code int `json:"code"`
Msg string `json:"msg"`
Data interface{} `json:"data"`
Code int `json:"code"`
Msg string `json:"msg"`
Data any `json:"data"`
}
func (r *Resp) parseError() (err error) {
+2 -2
View File
@@ -159,7 +159,7 @@ func (g *Client) singlePush(ctx context.Context, token, userID string, pushReq P
return g.request(ctx, pushURL, pushReq, token, nil)
}
func (g *Client) request(ctx context.Context, url string, input interface{}, token string, output interface{}) error {
func (g *Client) request(ctx context.Context, url string, input any, token string, output any) error {
header := map[string]string{"token": token}
resp := &Resp{}
resp.Data = output
@@ -170,7 +170,7 @@ func (g *Client) postReturn(
ctx context.Context,
url string,
header map[string]string,
input interface{},
input any,
output RespI,
timeout int,
) error {
@@ -23,7 +23,7 @@ const (
)
type Audience struct {
Object interface{}
Object any
audience map[string][]string
}
@@ -15,10 +15,10 @@
package body
type Message struct {
MsgContent string `json:"msg_content"`
Title string `json:"title,omitempty"`
ContentType string `json:"content_type,omitempty"`
Extras map[string]interface{} `json:"extras,omitempty"`
MsgContent string `json:"msg_content"`
Title string `json:"title,omitempty"`
ContentType string `json:"content_type,omitempty"`
Extras map[string]any `json:"extras,omitempty"`
}
func (m *Message) SetMsgContent(c string) {
@@ -33,9 +33,9 @@ func (m *Message) SetContentType(c string) {
m.ContentType = c
}
func (m *Message) SetExtras(key string, value interface{}) {
func (m *Message) SetExtras(key string, value any) {
if m.Extras == nil {
m.Extras = make(map[string]interface{})
m.Extras = make(map[string]any)
}
m.Extras[key] = value
}
@@ -29,7 +29,7 @@ const (
)
type Platform struct {
Os interface{}
Os any
osArry []string
}
@@ -15,11 +15,11 @@
package body
type PushObj struct {
Platform interface{} `json:"platform"`
Audience interface{} `json:"audience"`
Notification interface{} `json:"notification,omitempty"`
Message interface{} `json:"message,omitempty"`
Options interface{} `json:"options,omitempty"`
Platform any `json:"platform"`
Audience any `json:"audience"`
Notification any `json:"notification,omitempty"`
Message any `json:"message,omitempty"`
Options any `json:"options,omitempty"`
}
func (p *PushObj) SetPlatform(pf *Platform) {
+2 -2
View File
@@ -69,11 +69,11 @@ func (j *JPush) Push(ctx context.Context, userIDs []string, title, content strin
pushObj.SetNotification(&no)
pushObj.SetMessage(&msg)
pushObj.SetOptions(&opt)
var resp interface{}
var resp any
return j.request(ctx, pushObj, resp, 5)
}
func (j *JPush) request(ctx context.Context, po body.PushObj, resp interface{}, timeout int) error {
func (j *JPush) request(ctx context.Context, po body.PushObj, resp any, timeout int) error {
return http2.PostReturn(
ctx,
config.Config.Push.Jpns.PushUrl,
+1 -1
View File
@@ -133,7 +133,7 @@ func (p *Pusher) Push2User(ctx context.Context, userIDs []string, msg *sdkws.Msg
return nil
}
func (p *Pusher) UnmarshalNotificationElem(bytes []byte, t interface{}) error {
func (p *Pusher) UnmarshalNotificationElem(bytes []byte, t any) error {
var notification sdkws.NotificationElem
if err := json.Unmarshal(bytes, &notification); err != nil {
return err
+23 -18
View File
@@ -16,6 +16,12 @@ package conversation
import (
"context"
"errors"
"github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
"google.golang.org/grpc"
@@ -24,13 +30,11 @@ import (
"github.com/OpenIMSDK/tools/discoveryregistry"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/convert"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/controller"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/relation"
tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification"
@@ -43,24 +47,24 @@ type conversationServer struct {
}
func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error {
db, err := relation.NewGormDB()
if err != nil {
return err
}
if err := db.AutoMigrate(&tablerelation.ConversationModel{}); err != nil {
return err
}
rdb, err := cache.NewRedis()
if err != nil {
return err
}
conversationDB := relation.NewConversationGorm(db)
mongo, err := unrelation.NewMongo()
if err != nil {
return err
}
conversationDB, err := mgo.NewConversationMongo(mongo.GetDatabase())
if err != nil {
return err
}
groupRpcClient := rpcclient.NewGroupRpcClient(client)
msgRpcClient := rpcclient.NewMessageRpcClient(client)
pbconversation.RegisterConversationServer(server, &conversationServer{
conversationNotificationSender: notification.NewConversationNotificationSender(&msgRpcClient),
groupRpcClient: &groupRpcClient,
conversationDatabase: controller.NewConversationDatabase(conversationDB, cache.NewConversationRedis(rdb, cache.GetDefaultOpt(), conversationDB), tx.NewGorm(db)),
conversationDatabase: controller.NewConversationDatabase(conversationDB, cache.NewConversationRedis(rdb, cache.GetDefaultOpt(), conversationDB), tx.NewMongo(mongo.GetClient())),
})
return nil
}
@@ -145,7 +149,7 @@ func (c *conversationServer) SetConversations(ctx context.Context,
conversation.ConversationType = req.Conversation.ConversationType
conversation.UserID = req.Conversation.UserID
conversation.GroupID = req.Conversation.GroupID
m := make(map[string]interface{})
m := make(map[string]any)
if req.Conversation.RecvMsgOpt != nil {
m["recv_msg_opt"] = req.Conversation.RecvMsgOpt.Value
if req.Conversation.RecvMsgOpt.Value != conv.RecvMsgOpt {
@@ -229,11 +233,12 @@ func (c *conversationServer) SetConversations(ctx context.Context,
// 获取超级大群开启免打扰的用户ID.
func (c *conversationServer) GetRecvMsgNotNotifyUserIDs(ctx context.Context, req *pbconversation.GetRecvMsgNotNotifyUserIDsReq) (*pbconversation.GetRecvMsgNotNotifyUserIDsResp, error) {
userIDs, err := c.conversationDatabase.FindRecvMsgNotNotifyUserIDs(ctx, req.GroupID)
if err != nil {
return nil, err
}
return &pbconversation.GetRecvMsgNotNotifyUserIDsResp{UserIDs: userIDs}, nil
//userIDs, err := c.conversationDatabase.FindRecvMsgNotNotifyUserIDs(ctx, req.GroupID)
//if err != nil {
// return nil, err
//}
//return &pbconversation.GetRecvMsgNotNotifyUserIDsResp{UserIDs: userIDs}, nil
return nil, errors.New("deprecated")
}
// create conversation without notification for msg redis transfer.
@@ -284,7 +289,7 @@ func (c *conversationServer) CreateGroupChatConversations(ctx context.Context, r
func (c *conversationServer) SetConversationMaxSeq(ctx context.Context, req *pbconversation.SetConversationMaxSeqReq) (*pbconversation.SetConversationMaxSeqResp, error) {
if err := c.conversationDatabase.UpdateUsersConversationFiled(ctx, req.OwnerUserID, req.ConversationID,
map[string]interface{}{"max_seq": req.MaxSeq}); err != nil {
map[string]any{"max_seq": req.MaxSeq}); err != nil {
return nil, err
}
return &pbconversation.SetConversationMaxSeqResp{}, nil
+3 -20
View File
@@ -27,19 +27,11 @@ import (
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
)
func (s *friendServer) GetPaginationBlacks(
ctx context.Context,
req *pbfriend.GetPaginationBlacksReq,
) (resp *pbfriend.GetPaginationBlacksResp, err error) {
func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.GetPaginationBlacksReq) (resp *pbfriend.GetPaginationBlacksResp, err error) {
if err := s.userRpcClient.Access(ctx, req.UserID); err != nil {
return nil, err
}
var pageNumber, showNumber int32
if req.Pagination != nil {
pageNumber = req.Pagination.PageNumber
showNumber = req.Pagination.ShowNumber
}
blacks, total, err := s.blackDatabase.FindOwnerBlacks(ctx, req.UserID, pageNumber, showNumber)
total, blacks, err := s.blackDatabase.FindOwnerBlacks(ctx, req.UserID, req.Pagination)
if err != nil {
return nil, err
}
@@ -63,10 +55,7 @@ func (s *friendServer) IsBlack(ctx context.Context, req *pbfriend.IsBlackReq) (*
return resp, nil
}
func (s *friendServer) RemoveBlack(
ctx context.Context,
req *pbfriend.RemoveBlackReq,
) (*pbfriend.RemoveBlackResp, error) {
func (s *friendServer) RemoveBlack(ctx context.Context, req *pbfriend.RemoveBlackReq) (*pbfriend.RemoveBlackResp, error) {
if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil {
return nil, err
}
@@ -74,9 +63,6 @@ func (s *friendServer) RemoveBlack(
return nil, err
}
s.notificationSender.BlackDeletedNotification(ctx, req)
if err := CallbackAfterRemoveBlack(ctx, req); err != nil {
return nil, err
}
return &pbfriend.RemoveBlackResp{}, nil
}
@@ -88,9 +74,6 @@ func (s *friendServer) AddBlack(ctx context.Context, req *pbfriend.AddBlackReq)
if err != nil {
return nil, err
}
if err := CallbackBeforeAddBlack(ctx, req); err != nil {
return nil, err
}
black := relation.BlackModel{
OwnerUserID: req.OwnerUserID,
BlockUserID: req.BlackUserID,
+49 -61
View File
@@ -17,6 +17,8 @@ package friend
import (
"context"
"github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/protocol/sdkws"
"github.com/openimsdk/open-im-server/v3/pkg/authverify"
@@ -32,13 +34,13 @@ import (
pbfriend "github.com/OpenIMSDK/protocol/friend"
registry "github.com/OpenIMSDK/tools/discoveryregistry"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/controller"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/relation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification"
)
@@ -52,49 +54,65 @@ type friendServer struct {
}
func Start(client registry.SvcDiscoveryRegistry, server *grpc.Server) error {
db, err := relation.NewGormDB()
// Initialize MongoDB
mongo, err := unrelation.NewMongo()
if err != nil {
return err
}
if err := db.AutoMigrate(&tablerelation.FriendModel{}, &tablerelation.FriendRequestModel{}, &tablerelation.BlackModel{}); err != nil {
return err
}
// Initialize Redis
rdb, err := cache.NewRedis()
if err != nil {
return err
}
blackDB := relation.NewBlackGorm(db)
friendDB := relation.NewFriendGorm(db)
friendMongoDB, err := mgo.NewFriendMongo(mongo.GetDatabase())
if err != nil {
return err
}
friendRequestMongoDB, err := mgo.NewFriendRequestMongo(mongo.GetDatabase())
if err != nil {
return err
}
blackMongoDB, err := mgo.NewBlackMongo(mongo.GetDatabase())
if err != nil {
return err
}
// Initialize RPC clients
userRpcClient := rpcclient.NewUserRpcClient(client)
msgRpcClient := rpcclient.NewMessageRpcClient(client)
// Initialize notification sender
notificationSender := notification.NewFriendNotificationSender(
&msgRpcClient,
notification.WithRpcFunc(userRpcClient.GetUsersInfo),
)
// Register Friend server with refactored MongoDB and Redis integrations
pbfriend.RegisterFriendServer(server, &friendServer{
friendDatabase: controller.NewFriendDatabase(
friendDB,
relation.NewFriendRequestGorm(db),
cache.NewFriendCacheRedis(rdb, friendDB, cache.GetDefaultOpt()),
tx.NewGorm(db),
friendMongoDB,
friendRequestMongoDB,
cache.NewFriendCacheRedis(rdb, friendMongoDB, cache.GetDefaultOpt()),
tx.NewMongo(mongo.GetClient()),
),
blackDatabase: controller.NewBlackDatabase(
blackDB,
cache.NewBlackCacheRedis(rdb, blackDB, cache.GetDefaultOpt()),
blackMongoDB,
cache.NewBlackCacheRedis(rdb, blackMongoDB, cache.GetDefaultOpt()),
),
userRpcClient: &userRpcClient,
notificationSender: notificationSender,
RegisterCenter: client,
conversationRpcClient: rpcclient.NewConversationRpcClient(client),
})
return nil
}
// ok.
func (s *friendServer) ApplyToAddFriend(
ctx context.Context,
req *pbfriend.ApplyToAddFriendReq,
) (resp *pbfriend.ApplyToAddFriendResp, err error) {
func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *pbfriend.ApplyToAddFriendReq) (resp *pbfriend.ApplyToAddFriendResp, err error) {
defer log.ZInfo(ctx, utils.GetFuncName()+" Return")
resp = &pbfriend.ApplyToAddFriendResp{}
if err := authverify.CheckAccessV3(ctx, req.FromUserID); err != nil {
@@ -127,10 +145,7 @@ func (s *friendServer) ApplyToAddFriend(
}
// ok.
func (s *friendServer) ImportFriends(
ctx context.Context,
req *pbfriend.ImportFriendReq,
) (resp *pbfriend.ImportFriendResp, err error) {
func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFriendReq) (resp *pbfriend.ImportFriendResp, err error) {
defer log.ZInfo(ctx, utils.GetFuncName()+" Return")
if err := authverify.CheckAdmin(ctx); err != nil {
return nil, err
@@ -165,10 +180,7 @@ func (s *friendServer) ImportFriends(
}
// ok.
func (s *friendServer) RespondFriendApply(
ctx context.Context,
req *pbfriend.RespondFriendApplyReq,
) (resp *pbfriend.RespondFriendApplyResp, err error) {
func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.RespondFriendApplyReq) (resp *pbfriend.RespondFriendApplyResp, err error) {
defer log.ZInfo(ctx, utils.GetFuncName()+" Return")
resp = &pbfriend.RespondFriendApplyResp{}
if err := authverify.CheckAccessV3(ctx, req.ToUserID); err != nil {
@@ -204,10 +216,7 @@ func (s *friendServer) RespondFriendApply(
}
// ok.
func (s *friendServer) DeleteFriend(
ctx context.Context,
req *pbfriend.DeleteFriendReq,
) (resp *pbfriend.DeleteFriendResp, err error) {
func (s *friendServer) DeleteFriend(ctx context.Context, req *pbfriend.DeleteFriendReq) (resp *pbfriend.DeleteFriendResp, err error) {
defer log.ZInfo(ctx, utils.GetFuncName()+" Return")
resp = &pbfriend.DeleteFriendResp{}
if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil {
@@ -228,10 +237,7 @@ func (s *friendServer) DeleteFriend(
}
// ok.
func (s *friendServer) SetFriendRemark(
ctx context.Context,
req *pbfriend.SetFriendRemarkReq,
) (resp *pbfriend.SetFriendRemarkResp, err error) {
func (s *friendServer) SetFriendRemark(ctx context.Context, req *pbfriend.SetFriendRemarkReq) (resp *pbfriend.SetFriendRemarkResp, err error) {
defer log.ZInfo(ctx, utils.GetFuncName()+" Return")
if err = CallbackBeforeSetFriendRemark(ctx, req); err != nil && err != errs.ErrCallbackContinue {
@@ -256,10 +262,7 @@ func (s *friendServer) SetFriendRemark(
}
// ok.
func (s *friendServer) GetDesignatedFriends(
ctx context.Context,
req *pbfriend.GetDesignatedFriendsReq,
) (resp *pbfriend.GetDesignatedFriendsResp, err error) {
func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *pbfriend.GetDesignatedFriendsReq) (resp *pbfriend.GetDesignatedFriendsResp, err error) {
defer log.ZInfo(ctx, utils.GetFuncName()+" Return")
resp = &pbfriend.GetDesignatedFriendsResp{}
if utils.Duplicate(req.FriendUserIDs) {
@@ -290,15 +293,12 @@ func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context,
}
// ok 获取接收到的好友申请(即别人主动申请的).
func (s *friendServer) GetPaginationFriendsApplyTo(
ctx context.Context,
req *pbfriend.GetPaginationFriendsApplyToReq,
) (resp *pbfriend.GetPaginationFriendsApplyToResp, err error) {
func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *pbfriend.GetPaginationFriendsApplyToReq) (resp *pbfriend.GetPaginationFriendsApplyToResp, err error) {
defer log.ZInfo(ctx, utils.GetFuncName()+" Return")
if err := s.userRpcClient.Access(ctx, req.UserID); err != nil {
return nil, err
}
friendRequests, total, err := s.friendDatabase.PageFriendRequestToMe(ctx, req.UserID, req.Pagination.PageNumber, req.Pagination.ShowNumber)
total, friendRequests, err := s.friendDatabase.PageFriendRequestToMe(ctx, req.UserID, req.Pagination)
if err != nil {
return nil, err
}
@@ -312,16 +312,13 @@ func (s *friendServer) GetPaginationFriendsApplyTo(
}
// ok 获取主动发出去的好友申请列表.
func (s *friendServer) GetPaginationFriendsApplyFrom(
ctx context.Context,
req *pbfriend.GetPaginationFriendsApplyFromReq,
) (resp *pbfriend.GetPaginationFriendsApplyFromResp, err error) {
func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *pbfriend.GetPaginationFriendsApplyFromReq) (resp *pbfriend.GetPaginationFriendsApplyFromResp, err error) {
defer log.ZInfo(ctx, utils.GetFuncName()+" Return")
resp = &pbfriend.GetPaginationFriendsApplyFromResp{}
if err := s.userRpcClient.Access(ctx, req.UserID); err != nil {
return nil, err
}
friendRequests, total, err := s.friendDatabase.PageFriendRequestFromMe(ctx, req.UserID, req.Pagination.PageNumber, req.Pagination.ShowNumber)
total, friendRequests, err := s.friendDatabase.PageFriendRequestFromMe(ctx, req.UserID, req.Pagination)
if err != nil {
return nil, err
}
@@ -334,10 +331,7 @@ func (s *friendServer) GetPaginationFriendsApplyFrom(
}
// ok.
func (s *friendServer) IsFriend(
ctx context.Context,
req *pbfriend.IsFriendReq,
) (resp *pbfriend.IsFriendResp, err error) {
func (s *friendServer) IsFriend(ctx context.Context, req *pbfriend.IsFriendReq) (resp *pbfriend.IsFriendResp, err error) {
defer log.ZInfo(ctx, utils.GetFuncName()+" Return")
resp = &pbfriend.IsFriendResp{}
resp.InUser1Friends, resp.InUser2Friends, err = s.friendDatabase.CheckIn(ctx, req.UserID1, req.UserID2)
@@ -347,15 +341,12 @@ func (s *friendServer) IsFriend(
return resp, nil
}
func (s *friendServer) GetPaginationFriends(
ctx context.Context,
req *pbfriend.GetPaginationFriendsReq,
) (resp *pbfriend.GetPaginationFriendsResp, err error) {
func (s *friendServer) GetPaginationFriends(ctx context.Context, req *pbfriend.GetPaginationFriendsReq) (resp *pbfriend.GetPaginationFriendsResp, err error) {
defer log.ZInfo(ctx, utils.GetFuncName()+" Return")
if err := s.userRpcClient.Access(ctx, req.UserID); err != nil {
return nil, err
}
friends, total, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination.PageNumber, req.Pagination.ShowNumber)
total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination)
if err != nil {
return nil, err
}
@@ -368,10 +359,7 @@ func (s *friendServer) GetPaginationFriends(
return resp, nil
}
func (s *friendServer) GetFriendIDs(
ctx context.Context,
req *pbfriend.GetFriendIDsReq,
) (resp *pbfriend.GetFriendIDsResp, err error) {
func (s *friendServer) GetFriendIDs(ctx context.Context, req *pbfriend.GetFriendIDsReq) (resp *pbfriend.GetFriendIDsResp, err error) {
defer log.ZInfo(ctx, utils.GetFuncName()+" Return")
if err := s.userRpcClient.Access(ctx, req.UserID); err != nil {
return nil, err
+2 -2
View File
@@ -26,7 +26,7 @@ func (s *groupServer) GetGroupInfoCache(
ctx context.Context,
req *pbgroup.GetGroupInfoCacheReq,
) (resp *pbgroup.GetGroupInfoCacheResp, err error) {
group, err := s.GroupDatabase.TakeGroup(ctx, req.GroupID)
group, err := s.db.TakeGroup(ctx, req.GroupID)
if err != nil {
return nil, err
}
@@ -38,7 +38,7 @@ func (s *groupServer) GetGroupMemberCache(
ctx context.Context,
req *pbgroup.GetGroupMemberCacheReq,
) (resp *pbgroup.GetGroupMemberCacheResp, err error) {
members, err := s.GroupDatabase.TakeGroupMember(ctx, req.GroupID, req.GroupMemberID)
members, err := s.db.TakeGroupMember(ctx, req.GroupID, req.GroupMemberID)
if err != nil {
return nil, err
}
+1 -1
View File
@@ -27,7 +27,7 @@ import (
func UpdateGroupInfoMap(ctx context.Context, group *sdkws.GroupInfoForSet) map[string]any {
m := make(map[string]any)
if group.GroupName != "" {
m["name"] = group.GroupName
m["group_name"] = group.GroupName
}
if group.Notification != "" {
m["notification"] = group.Notification
+2 -112
View File
@@ -17,119 +17,9 @@ package group
import (
"context"
"github.com/OpenIMSDK/tools/utils"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
)
func (s *groupServer) FindGroupMember(ctx context.Context, groupIDs []string, userIDs []string, roleLevels []int32) ([]*relationtb.GroupMemberModel, error) {
members, err := s.GroupDatabase.FindGroupMember(ctx, groupIDs, userIDs, roleLevels)
if err != nil {
return nil, err
}
emptyUserIDs := make(map[string]struct{})
for _, member := range members {
if member.Nickname == "" || member.FaceURL == "" {
emptyUserIDs[member.UserID] = struct{}{}
}
}
if len(emptyUserIDs) > 0 {
users, err := s.User.GetPublicUserInfoMap(ctx, utils.Keys(emptyUserIDs), true)
if err != nil {
return nil, err
}
for i, member := range members {
user, ok := users[member.UserID]
if !ok {
continue
}
if member.Nickname == "" {
members[i].Nickname = user.Nickname
}
if member.FaceURL == "" {
members[i].FaceURL = user.FaceURL
}
}
}
return members, nil
}
func (s *groupServer) TakeGroupMember(
ctx context.Context,
groupID string,
userID string,
) (*relationtb.GroupMemberModel, error) {
member, err := s.GroupDatabase.TakeGroupMember(ctx, groupID, userID)
if err != nil {
return nil, err
}
if member.Nickname == "" || member.FaceURL == "" {
user, err := s.User.GetPublicUserInfo(ctx, userID)
if err != nil {
return nil, err
}
if member.Nickname == "" {
member.Nickname = user.Nickname
}
if member.FaceURL == "" {
member.FaceURL = user.FaceURL
}
}
return member, nil
}
func (s *groupServer) TakeGroupOwner(ctx context.Context, groupID string) (*relationtb.GroupMemberModel, error) {
owner, err := s.GroupDatabase.TakeGroupOwner(ctx, groupID)
if err != nil {
return nil, err
}
if owner.Nickname == "" || owner.FaceURL == "" {
user, err := s.User.GetUserInfo(ctx, owner.UserID)
if err != nil {
return nil, err
}
if owner.Nickname == "" {
owner.Nickname = user.Nickname
}
if owner.FaceURL == "" {
owner.FaceURL = user.FaceURL
}
}
return owner, nil
}
func (s *groupServer) PageGetGroupMember(
ctx context.Context,
groupID string,
pageNumber, showNumber int32,
) (uint32, []*relationtb.GroupMemberModel, error) {
total, members, err := s.GroupDatabase.PageGetGroupMember(ctx, groupID, pageNumber, showNumber)
if err != nil {
return 0, nil, err
}
emptyUserIDs := make(map[string]struct{})
for _, member := range members {
if member.Nickname == "" || member.FaceURL == "" {
emptyUserIDs[member.UserID] = struct{}{}
}
}
if len(emptyUserIDs) > 0 {
users, err := s.User.GetPublicUserInfoMap(ctx, utils.Keys(emptyUserIDs), true)
if err != nil {
return 0, nil, err
}
for i, member := range members {
user, ok := users[member.UserID]
if !ok {
continue
}
if member.Nickname == "" {
members[i].Nickname = user.Nickname
}
if member.FaceURL == "" {
members[i].FaceURL = user.FaceURL
}
}
}
return total, members, nil
func (s *groupServer) PopulateGroupMember(ctx context.Context, members ...*relationtb.GroupMemberModel) error {
return s.Notification.PopulateGroupMember(ctx, members...)
}
File diff suppressed because it is too large Load Diff
+3 -3
View File
@@ -26,16 +26,16 @@ func (s *groupServer) GroupCreateCount(ctx context.Context, req *group.GroupCrea
if req.Start > req.End {
return nil, errs.ErrArgs.Wrap("start > end")
}
total, err := s.GroupDatabase.CountTotal(ctx, nil)
total, err := s.db.CountTotal(ctx, nil)
if err != nil {
return nil, err
}
start := time.UnixMilli(req.Start)
before, err := s.GroupDatabase.CountTotal(ctx, &start)
before, err := s.db.CountTotal(ctx, &start)
if err != nil {
return nil, err
}
count, err := s.GroupDatabase.CountRangeEverydayTotal(ctx, start, time.UnixMilli(req.End))
count, err := s.db.CountRangeEverydayTotal(ctx, start, time.UnixMilli(req.End))
if err != nil {
return nil, err
}
+5 -89
View File
@@ -16,99 +16,15 @@ package group
import (
"context"
"fmt"
"strings"
"errors"
"github.com/OpenIMSDK/protocol/constant"
pbgroup "github.com/OpenIMSDK/protocol/group"
sdkws "github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/convert"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
)
func (s *groupServer) GetJoinedSuperGroupList(
ctx context.Context,
req *pbgroup.GetJoinedSuperGroupListReq,
) (*pbgroup.GetJoinedSuperGroupListResp, error) {
resp := &pbgroup.GetJoinedSuperGroupListResp{}
groupIDs, err := s.GroupDatabase.FindJoinSuperGroup(ctx, req.UserID)
if err != nil {
return nil, err
}
if len(groupIDs) == 0 {
return resp, nil
}
owners, err := s.FindGroupMember(ctx, groupIDs, nil, []int32{constant.GroupOwner})
if err != nil {
return nil, err
}
ownerMap := utils.SliceToMap(owners, func(e *relation.GroupMemberModel) string {
return e.GroupID
})
if ids := utils.Single(groupIDs, utils.Keys(ownerMap)); len(ids) > 0 {
return nil, errs.ErrData.Wrap(fmt.Sprintf("super group %s not owner", strings.Join(ids, ",")))
}
groups, err := s.GroupDatabase.FindGroup(ctx, groupIDs)
if err != nil {
return nil, err
}
groupMap := utils.SliceToMap(groups, func(e *relation.GroupModel) string {
return e.GroupID
})
if ids := utils.Single(groupIDs, utils.Keys(groupMap)); len(ids) > 0 {
return nil, errs.ErrData.Wrap(fmt.Sprintf("super group info %s not found", strings.Join(ids, ",")))
}
superGroupMembers, err := s.GroupDatabase.FindSuperGroup(ctx, groupIDs)
if err != nil {
return nil, err
}
superGroupMemberMap := utils.SliceToMapAny(
superGroupMembers,
func(e *unrelation.SuperGroupModel) (string, []string) {
return e.GroupID, e.MemberIDs
},
)
resp.Groups = utils.Slice(groupIDs, func(groupID string) *sdkws.GroupInfo {
return convert.Db2PbGroupInfo(groupMap[groupID], ownerMap[groupID].UserID, uint32(len(superGroupMemberMap)))
})
return resp, nil
func (s *groupServer) GetJoinedSuperGroupList(context.Context, *pbgroup.GetJoinedSuperGroupListReq) (*pbgroup.GetJoinedSuperGroupListResp, error) {
return nil, errors.New("deprecated")
}
func (s *groupServer) GetSuperGroupsInfo(
ctx context.Context,
req *pbgroup.GetSuperGroupsInfoReq,
) (resp *pbgroup.GetSuperGroupsInfoResp, err error) {
resp = &pbgroup.GetSuperGroupsInfoResp{}
if len(req.GroupIDs) == 0 {
return nil, errs.ErrArgs.Wrap("groupIDs empty")
}
groups, err := s.GroupDatabase.FindGroup(ctx, req.GroupIDs)
if err != nil {
return nil, err
}
superGroupMembers, err := s.GroupDatabase.FindSuperGroup(ctx, req.GroupIDs)
if err != nil {
return nil, err
}
superGroupMemberMap := utils.SliceToMapAny(
superGroupMembers,
func(e *unrelation.SuperGroupModel) (string, []string) {
return e.GroupID, e.MemberIDs
},
)
owners, err := s.FindGroupMember(ctx, req.GroupIDs, nil, []int32{constant.GroupOwner})
if err != nil {
return nil, err
}
ownerMap := utils.SliceToMap(owners, func(e *relation.GroupMemberModel) string {
return e.GroupID
})
resp.GroupInfos = utils.Slice(groups, func(e *relation.GroupModel) *sdkws.GroupInfo {
return convert.Db2PbGroupInfo(e, ownerMap[e.GroupID].UserID, uint32(len(superGroupMemberMap[e.GroupID])))
})
return resp, nil
func (s *groupServer) GetSuperGroupsInfo(context.Context, *pbgroup.GetSuperGroupsInfoReq) (resp *pbgroup.GetSuperGroupsInfoResp, err error) {
return nil, errors.New("deprecated")
}
+3 -4
View File
@@ -15,12 +15,11 @@
package msg
import (
"github.com/redis/go-redis/v9"
"gorm.io/gorm"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/utils"
"github.com/redis/go-redis/v9"
"go.mongodb.org/mongo-driver/mongo"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
)
@@ -45,7 +44,7 @@ func isMessageHasReadEnabled(msgData *sdkws.MsgData) bool {
func IsNotFound(err error) bool {
switch utils.Unwrap(err) {
case redis.Nil, gorm.ErrRecordNotFound:
case redis.Nil, mongo.ErrNoDocuments:
return true
default:
return false
+11 -13
View File
@@ -32,11 +32,11 @@ func genLogID() string {
}
func (t *thirdServer) UploadLogs(ctx context.Context, req *third.UploadLogsReq) (*third.UploadLogsResp, error) {
var DBlogs []*relationtb.Log
var DBlogs []*relationtb.LogModel
userID := ctx.Value(constant.OpUserID).(string)
platform := constant.PlatformID2Name[int(req.Platform)]
for _, fileURL := range req.FileURLs {
log := relationtb.Log{
log := relationtb.LogModel{
Version: req.Version,
SystemType: req.SystemType,
Platform: platform,
@@ -57,7 +57,7 @@ func (t *thirdServer) UploadLogs(ctx context.Context, req *third.UploadLogsReq)
}
}
if log.LogID == "" {
return nil, errs.ErrData.Wrap("Log id gen error")
return nil, errs.ErrData.Wrap("LogModel id gen error")
}
DBlogs = append(DBlogs, &log)
}
@@ -92,8 +92,8 @@ func (t *thirdServer) DeleteLogs(ctx context.Context, req *third.DeleteLogsReq)
return &third.DeleteLogsResp{}, nil
}
func dbToPbLogInfos(logs []*relationtb.Log) []*third.LogInfo {
db2pbForLogInfo := func(log *relationtb.Log) *third.LogInfo {
func dbToPbLogInfos(logs []*relationtb.LogModel) []*third.LogInfo {
db2pbForLogInfo := func(log *relationtb.LogModel) *third.LogInfo {
return &third.LogInfo{
Filename: log.FileName,
UserID: log.UserID,
@@ -120,7 +120,7 @@ func (t *thirdServer) SearchLogs(ctx context.Context, req *third.SearchLogsReq)
if req.StartTime > req.EndTime {
return nil, errs.ErrArgs.Wrap("startTime>endTime")
}
total, logs, err := t.thirdDatabase.SearchLogs(ctx, req.Keyword, time.UnixMilli(req.StartTime), time.UnixMilli(req.EndTime), req.Pagination.PageNumber, req.Pagination.ShowNumber)
total, logs, err := t.thirdDatabase.SearchLogs(ctx, req.Keyword, time.UnixMilli(req.StartTime), time.UnixMilli(req.EndTime), req.Pagination)
if err != nil {
return nil, err
}
@@ -128,18 +128,16 @@ func (t *thirdServer) SearchLogs(ctx context.Context, req *third.SearchLogsReq)
for _, log := range logs {
userIDs = append(userIDs, log.UserID)
}
users, err := t.thirdDatabase.FindUsers(ctx, userIDs)
userMap, err := t.userRpcClient.GetUsersInfoMap(ctx, userIDs)
if err != nil {
return nil, err
}
IDtoName := make(map[string]string)
for _, user := range users {
IDtoName[user.UserID] = user.Nickname
}
for _, pbLog := range pbLogs {
pbLog.Nickname = IDtoName[pbLog.UserID]
if user, ok := userMap[pbLog.UserID]; ok {
pbLog.Nickname = user.Nickname
}
}
resp.LogsInfos = pbLogs
resp.Total = total
resp.Total = uint32(total)
return &resp, nil
}
+2 -2
View File
@@ -64,7 +64,7 @@ func (t *thirdServer) InitiateMultipartUpload(ctx context.Context, req *third.In
Key: haErr.Object.Key,
Size: haErr.Object.Size,
ContentType: req.ContentType,
Cause: req.Cause,
Group: req.Cause,
CreateTime: time.Now(),
}
if err := t.s3dataBase.SetObject(ctx, obj); err != nil {
@@ -143,7 +143,7 @@ func (t *thirdServer) CompleteMultipartUpload(ctx context.Context, req *third.Co
Key: result.Key,
Size: result.Size,
ContentType: req.ContentType,
Cause: req.Cause,
Group: req.Cause,
CreateTime: time.Now(),
}
if err := t.s3dataBase.SetObject(ctx, obj); err != nil {
+18 -22
View File
@@ -17,15 +17,17 @@ package third
import (
"context"
"fmt"
"net/url"
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/kodo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/oss"
"google.golang.org/grpc"
"github.com/OpenIMSDK/protocol/third"
@@ -34,13 +36,22 @@ import (
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/controller"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/relation"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
)
func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error {
mongo, err := unrelation.NewMongo()
if err != nil {
return err
}
logdb, err := mgo.NewLogMongo(mongo.GetDatabase())
if err != nil {
return err
}
s3db, err := mgo.NewS3Mongo(mongo.GetDatabase())
if err != nil {
return err
}
apiURL := config.Config.Object.ApiURL
if apiURL == "" {
return fmt.Errorf("api url is empty")
@@ -56,13 +67,6 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
if err != nil {
return err
}
db, err := relation.NewGormDB()
if err != nil {
return err
}
if err := db.AutoMigrate(&relationtb.ObjectModel{}); err != nil {
return err
}
// 根据配置文件策略选择 oss 方式
enable := config.Config.Object.Enable
var o s3.Interface
@@ -73,25 +77,17 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
o, err = cos.NewCos()
case "oss":
o, err = oss.NewOSS()
case "kodo":
o, err = kodo.NewKodo()
default:
err = fmt.Errorf("invalid object enable: %s", enable)
}
if err != nil {
return err
}
//specialerror.AddErrHandler(func(err error) errs.CodeError {
// if o.IsNotFound(err) {
// return errs.ErrRecordNotFound
// }
// return nil
//})
third.RegisterThirdServer(server, &thirdServer{
apiURL: apiURL,
thirdDatabase: controller.NewThirdDatabase(cache.NewMsgCacheModel(rdb), db),
thirdDatabase: controller.NewThirdDatabase(cache.NewMsgCacheModel(rdb), logdb),
userRpcClient: rpcclient.NewUserRpcClient(client),
s3dataBase: controller.NewS3Database(rdb, o, relation.NewObjectInfo(db)),
s3dataBase: controller.NewS3Database(rdb, o, s3db),
defaultExpire: time.Hour * 24 * 7,
})
return nil
+16 -27
View File
@@ -20,11 +20,14 @@ import (
"strings"
"time"
"github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/authverify"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
@@ -35,7 +38,6 @@ import (
"github.com/openimsdk/open-im-server/v3/pkg/common/convert"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/controller"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/relation"
tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification"
@@ -55,10 +57,6 @@ type userServer struct {
}
func Start(client registry.SvcDiscoveryRegistry, server *grpc.Server) error {
db, err := relation.NewGormDB()
if err != nil {
return err
}
rdb, err := cache.NewRedis()
if err != nil {
return err
@@ -67,9 +65,6 @@ func Start(client registry.SvcDiscoveryRegistry, server *grpc.Server) error {
if err != nil {
return err
}
if err := db.AutoMigrate(&tablerelation.UserModel{}); err != nil {
return err
}
users := make([]*tablerelation.UserModel, 0)
if len(config.Config.Manager.UserID) != len(config.Config.Manager.Nickname) {
return errors.New("len(config.Config.Manager.AppManagerUid) != len(config.Config.Manager.Nickname)")
@@ -77,10 +72,13 @@ func Start(client registry.SvcDiscoveryRegistry, server *grpc.Server) error {
for k, v := range config.Config.Manager.UserID {
users = append(users, &tablerelation.UserModel{UserID: v, Nickname: config.Config.Manager.Nickname[k], AppMangerLevel: constant.AppAdmin})
}
userDB := relation.NewUserGorm(db)
userDB, err := mgo.NewUserMongo(mongo.GetDatabase())
if err != nil {
return err
}
cache := cache.NewUserCacheRedis(rdb, userDB, cache.GetDefaultOpt())
userMongoDB := unrelation.NewUserMongoDriver(mongo.GetDatabase())
database := controller.NewUserDatabase(userDB, cache, tx.NewGorm(db), userMongoDB)
database := controller.NewUserDatabase(userDB, cache, tx.NewMongo(mongo.GetClient()), userMongoDB)
friendRpcClient := rpcclient.NewFriendRpcClient(client)
groupRpcClient := rpcclient.NewGroupRpcClient(client)
msgRpcClient := rpcclient.NewMessageRpcClient(client)
@@ -118,12 +116,8 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI
if err := CallbackBeforeUpdateUserInfo(ctx, req); err != nil {
return nil, err
}
user := convert.UserPb2DB(req.UserInfo)
if err != nil {
return nil, err
}
err = s.Update(ctx, user)
if err != nil {
data := convert.UserPb2DBMap(req.UserInfo)
if err := s.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil {
return nil, err
}
_ = s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID)
@@ -153,7 +147,7 @@ func (s *userServer) SetGlobalRecvMessageOpt(ctx context.Context, req *pbuser.Se
if _, err := s.FindWithError(ctx, []string{req.UserID}); err != nil {
return nil, err
}
m := make(map[string]interface{}, 1)
m := make(map[string]any, 1)
m["global_recv_msg_opt"] = req.GlobalRecvMsgOpt
if err := s.UpdateByMap(ctx, req.UserID, m); err != nil {
return nil, err
@@ -175,7 +169,7 @@ func (s *userServer) AccountCheck(ctx context.Context, req *pbuser.AccountCheckR
if err != nil {
return nil, err
}
userIDs := make(map[string]interface{}, 0)
userIDs := make(map[string]any, 0)
for _, v := range users {
userIDs[v.UserID] = nil
}
@@ -192,12 +186,7 @@ func (s *userServer) AccountCheck(ctx context.Context, req *pbuser.AccountCheckR
}
func (s *userServer) GetPaginationUsers(ctx context.Context, req *pbuser.GetPaginationUsersReq) (resp *pbuser.GetPaginationUsersResp, err error) {
var pageNumber, showNumber int32
if req.Pagination != nil {
pageNumber = req.Pagination.PageNumber
showNumber = req.Pagination.ShowNumber
}
users, total, err := s.Page(ctx, pageNumber, showNumber)
total, users, err := s.Page(ctx, req.Pagination)
if err != nil {
return nil, err
}
@@ -269,11 +258,11 @@ func (s *userServer) GetGlobalRecvMessageOpt(ctx context.Context, req *pbuser.Ge
// GetAllUserID Get user account by page.
func (s *userServer) GetAllUserID(ctx context.Context, req *pbuser.GetAllUserIDReq) (resp *pbuser.GetAllUserIDResp, err error) {
userIDs, err := s.UserDatabase.GetAllUserID(ctx, req.Pagination.PageNumber, req.Pagination.ShowNumber)
total, userIDs, err := s.UserDatabase.GetAllUserID(ctx, req.Pagination)
if err != nil {
return nil, err
}
return &pbuser.GetAllUserIDResp{UserIDs: userIDs}, nil
return &pbuser.GetAllUserIDResp{Total: int32(total), UserIDs: userIDs}, nil
}
// SubscribeOrCancelUsersStatus Subscribe online or cancel online users.
+8 -2
View File
@@ -19,6 +19,8 @@ import (
"math/rand"
"time"
"github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/mcontext"
"github.com/OpenIMSDK/tools/utils"
@@ -91,7 +93,11 @@ func (c *MsgTool) ConversationsDestructMsgs() {
}
for i := 0; i < count; i++ {
pageNumber := rand.Int63() % maxPage
conversationIDs, err := c.conversationDatabase.PageConversationIDs(ctx, int32(pageNumber), batchNum)
pagination := &sdkws.RequestPagination{
PageNumber: int32(pageNumber),
ShowNumber: batchNum,
}
conversationIDs, err := c.conversationDatabase.PageConversationIDs(ctx, pagination)
if err != nil {
log.ZError(ctx, "PageConversationIDs failed", err, "pageNumber", pageNumber)
continue
@@ -133,7 +139,7 @@ func (c *MsgTool) ConversationsDestructMsgs() {
continue
}
if len(seqs) > 0 {
if err := c.conversationDatabase.UpdateUsersConversationFiled(ctx, []string{conversation.OwnerUserID}, conversation.ConversationID, map[string]interface{}{"latest_msg_destruct_time": now}); err != nil {
if err := c.conversationDatabase.UpdateUsersConversationFiled(ctx, []string{conversation.OwnerUserID}, conversation.ConversationID, map[string]any{"latest_msg_destruct_time": now}); err != nil {
log.ZError(ctx, "updateUsersConversationFiled failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID)
continue
}
+37 -18
View File
@@ -19,6 +19,11 @@ import (
"fmt"
"math"
"github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/redis/go-redis/v9"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
@@ -31,13 +36,11 @@ import (
"github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/mcontext"
"github.com/OpenIMSDK/tools/mw"
"github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/controller"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/relation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification"
@@ -72,33 +75,45 @@ func InitMsgTool() (*MsgTool, error) {
if err != nil {
return nil, err
}
db, err := relation.NewGormDB()
if err != nil {
return nil, err
}
discov, err := kdisc.NewDiscoveryRegister(config.Config.Envs.Discovery)
/*
discov, err := zookeeper.NewClient(config.Config.Zookeeper.ZkAddr, config.Config.Zookeeper.Schema,
zookeeper.WithFreq(time.Hour), zookeeper.WithRoundRobin(), zookeeper.WithUserNameAndPassword(config.Config.Zookeeper.Username,
config.Config.Zookeeper.Password), zookeeper.WithTimeout(10), zookeeper.WithLogger(log.NewZkLogger()))*/
if err != nil {
return nil, err
}
discov.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()))
userDB := relation.NewUserGorm(db)
userDB, err := mgo.NewUserMongo(mongo.GetDatabase())
if err != nil {
return nil, err
}
msgDatabase := controller.InitCommonMsgDatabase(rdb, mongo.GetDatabase())
userMongoDB := unrelation.NewUserMongoDriver(mongo.GetDatabase())
ctxTx := tx.NewMongo(mongo.GetClient())
userDatabase := controller.NewUserDatabase(
userDB,
cache.NewUserCacheRedis(rdb, relation.NewUserGorm(db), cache.GetDefaultOpt()),
tx.NewGorm(db),
cache.NewUserCacheRedis(rdb, userDB, cache.GetDefaultOpt()),
ctxTx,
userMongoDB,
)
groupDatabase := controller.InitGroupDatabase(db, rdb, mongo.GetDatabase(), nil)
groupDB, err := mgo.NewGroupMongo(mongo.GetDatabase())
if err != nil {
return nil, err
}
groupMemberDB, err := mgo.NewGroupMember(mongo.GetDatabase())
if err != nil {
return nil, err
}
groupRequestDB, err := mgo.NewGroupRequestMgo(mongo.GetDatabase())
if err != nil {
return nil, err
}
conversationDB, err := mgo.NewConversationMongo(mongo.GetDatabase())
if err != nil {
return nil, err
}
groupDatabase := controller.NewGroupDatabase(rdb, groupDB, groupMemberDB, groupRequestDB, ctxTx, nil)
conversationDatabase := controller.NewConversationDatabase(
relation.NewConversationGorm(db),
cache.NewConversationRedis(rdb, cache.GetDefaultOpt(), relation.NewConversationGorm(db)),
tx.NewGorm(db),
conversationDB,
cache.NewConversationRedis(rdb, cache.GetDefaultOpt(), conversationDB),
ctxTx,
)
msgRpcClient := rpcclient.NewMessageRpcClient(discov)
msgNotificationSender := notification.NewMsgNotificationSender(rpcclient.WithRpcClient(&msgRpcClient))
@@ -144,7 +159,11 @@ func (c *MsgTool) AllConversationClearMsgAndFixSeq() {
}
for i := 0; i < count; i++ {
pageNumber := rand.Int63() % maxPage
conversationIDs, err := c.conversationDatabase.PageConversationIDs(ctx, int32(pageNumber), batchNum)
pagination := &sdkws.RequestPagination{
PageNumber: int32(pageNumber),
ShowNumber: batchNum,
}
conversationIDs, err := c.conversationDatabase.PageConversationIDs(ctx, pagination)
if err != nil {
log.ZError(ctx, "PageConversationIDs failed", err, "pageNumber", pageNumber)
continue
+1 -1
View File
@@ -32,7 +32,7 @@ func (c *MsgTool) convertTools() {
for _, conversationID := range conversationIDs {
conversationIDs = append(conversationIDs, msgprocessor.GetNotificationConversationIDByConversationID(conversationID))
}
userIDs, err := c.userDatabase.GetAllUserID(ctx, 0, 0)
_, userIDs, err := c.userDatabase.GetAllUserID(ctx, nil)
if err != nil {
log.ZError(ctx, "get all user ids failed", err)
return