Merge remote-tracking branch 'origin/main' into pre-release-v3.8.4

# Conflicts:
#	.env
#	.github/workflows/docker-build-and-release-services-images.yml
#	.github/workflows/merge-from-milestone.yml
#	.github/workflows/update-version-file-on-release.yml
#	README.md
#	README_zh_CN.md
#	cmd/main.go
#	config/discovery.yml
#	config/notification.yml
#	config/openim-api.yml
#	config/openim-msggateway.yml
#	config/openim-msgtransfer.yml
#	config/openim-push.yml
#	config/openim-rpc-auth.yml
#	config/openim-rpc-conversation.yml
#	config/openim-rpc-friend.yml
#	config/openim-rpc-group.yml
#	config/openim-rpc-msg.yml
#	config/openim-rpc-third.yml
#	config/openim-rpc-user.yml
#	config/share.yml
#	config/webhooks.yml
#	deployments/templates/config.yaml
#	docker-compose.yml
#	go.mod
#	go.sum
#	internal/api/init.go
#	internal/api/jssdk/tools.go
#	internal/api/msg.go
#	internal/api/prometheus_discovery.go
#	internal/api/router.go
#	internal/msggateway/init.go
#	internal/msggateway/ws_server.go
#	internal/msgtransfer/init.go
#	internal/msgtransfer/online_history_msg_handler.go
#	internal/msgtransfer/online_msg_to_mongo_handler.go
#	internal/push/push.go
#	internal/rpc/auth/auth.go
#	internal/rpc/conversation/conversation.go
#	internal/rpc/group/group.go
#	internal/rpc/msg/callback.go
#	internal/rpc/msg/server.go
#	internal/rpc/relation/friend.go
#	internal/rpc/relation/notification.go
#	internal/rpc/third/third.go
#	internal/rpc/user/user.go
#	internal/tools/cron/cron_task.go
#	magefile.go
#	pkg/common/cmd/api.go
#	pkg/common/cmd/msg_transfer.go
#	pkg/common/config/config.go
#	pkg/common/discovery/discoveryregister.go
#	pkg/common/prommetrics/prommetrics.go
#	pkg/common/startrpc/start.go
#	pkg/common/storage/database/mgo/cache.go
#	pkg/common/storage/database/mgo/msg_test.go
#	pkg/rpcli/auth.go
#	pkg/rpcli/tool.go
#	pkg/rpcli/user.go
#	test/stress-test-v2/main.go
#	test/stress-test/main.go
#	tools/seq/internal/seq.go
#	version/version

pkg/common/cmd/api.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/openimsdk/open-im-server/v3/internal/api"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/startrpc"
 	"github.com/openimsdk/open-im-server/v3/version"
 	"github.com/openimsdk/tools/system/program"
@@ -84,7 +85,7 @@ func (a *ApiCmd) runE() error {
 		a.apiConfig.API.Api.ListenIP, "",
 		a.apiConfig.API.Prometheus.AutoSetPorts,
 		nil, int(a.apiConfig.Index),
-		a.apiConfig.Discovery.RpcService.MessageGateway,
+		prommetrics.APIKeyName,
 		&a.apiConfig.Notification,
 		a.apiConfig,
 		[]string{},

pkg/common/cmd/constant.go

@@ -1,96 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package cmd
-
-import (
-	"strings"
-)
-
-var (
-	FileName                         string
-	NotificationFileName             string
-	ShareFileName                    string
-	WebhooksConfigFileName           string
-	LocalCacheConfigFileName         string
-	KafkaConfigFileName              string
-	RedisConfigFileName              string
-	MongodbConfigFileName            string
-	MinioConfigFileName              string
-	LogConfigFileName                string
-	OpenIMAPICfgFileName             string
-	OpenIMCronTaskCfgFileName        string
-	OpenIMMsgGatewayCfgFileName      string
-	OpenIMMsgTransferCfgFileName     string
-	OpenIMPushCfgFileName            string
-	OpenIMRPCAuthCfgFileName         string
-	OpenIMRPCConversationCfgFileName string
-	OpenIMRPCFriendCfgFileName       string
-	OpenIMRPCGroupCfgFileName        string
-	OpenIMRPCMsgCfgFileName          string
-	OpenIMRPCThirdCfgFileName        string
-	OpenIMRPCUserCfgFileName         string
-	DiscoveryConfigFilename          string
-)
-
-var ConfigEnvPrefixMap map[string]string
-
-func init() {
-	FileName = "config.yaml"
-	NotificationFileName = "notification.yml"
-	ShareFileName = "share.yml"
-	WebhooksConfigFileName = "webhooks.yml"
-	LocalCacheConfigFileName = "local-cache.yml"
-	KafkaConfigFileName = "kafka.yml"
-	RedisConfigFileName = "redis.yml"
-	MongodbConfigFileName = "mongodb.yml"
-	MinioConfigFileName = "minio.yml"
-	LogConfigFileName = "log.yml"
-	OpenIMAPICfgFileName = "openim-api.yml"
-	OpenIMCronTaskCfgFileName = "openim-crontask.yml"
-	OpenIMMsgGatewayCfgFileName = "openim-msggateway.yml"
-	OpenIMMsgTransferCfgFileName = "openim-msgtransfer.yml"
-	OpenIMPushCfgFileName = "openim-push.yml"
-	OpenIMRPCAuthCfgFileName = "openim-rpc-auth.yml"
-	OpenIMRPCConversationCfgFileName = "openim-rpc-conversation.yml"
-	OpenIMRPCFriendCfgFileName = "openim-rpc-friend.yml"
-	OpenIMRPCGroupCfgFileName = "openim-rpc-group.yml"
-	OpenIMRPCMsgCfgFileName = "openim-rpc-msg.yml"
-	OpenIMRPCThirdCfgFileName = "openim-rpc-third.yml"
-	OpenIMRPCUserCfgFileName = "openim-rpc-user.yml"
-	DiscoveryConfigFilename = "discovery.yml"
-
-	ConfigEnvPrefixMap = make(map[string]string)
-	fileNames := []string{
-		FileName, NotificationFileName, ShareFileName, WebhooksConfigFileName,
-		KafkaConfigFileName, RedisConfigFileName,
-		MongodbConfigFileName, MinioConfigFileName, LogConfigFileName,
-		OpenIMAPICfgFileName, OpenIMCronTaskCfgFileName, OpenIMMsgGatewayCfgFileName,
-		OpenIMMsgTransferCfgFileName, OpenIMPushCfgFileName, OpenIMRPCAuthCfgFileName,
-		OpenIMRPCConversationCfgFileName, OpenIMRPCFriendCfgFileName, OpenIMRPCGroupCfgFileName,
-		OpenIMRPCMsgCfgFileName, OpenIMRPCThirdCfgFileName, OpenIMRPCUserCfgFileName, DiscoveryConfigFilename,
-	}
-
-	for _, fileName := range fileNames {
-		envKey := strings.TrimSuffix(strings.TrimSuffix(fileName, ".yml"), ".yaml")
-		envKey = "IMENV_" + envKey
-		envKey = strings.ToUpper(strings.ReplaceAll(envKey, "-", "_"))
-		ConfigEnvPrefixMap[fileName] = envKey
-	}
-}
-
-const (
-	FlagConf          = "config_folder_path"
-	FlagTransferIndex = "index"
-)

pkg/common/cmd/msg_transfer.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/openimsdk/open-im-server/v3/internal/msgtransfer"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/startrpc"
 	"github.com/openimsdk/open-im-server/v3/version"
 	"github.com/openimsdk/tools/system/program"
@@ -65,7 +66,7 @@ func (m *MsgTransferCmd) runE() error {
 		"", "",
 		true,
 		nil, int(m.msgTransferConfig.Index),
-		"",
+		prommetrics.MessageTransferKeyName,
 		nil,
 		m.msgTransferConfig,
 		[]string{},

pkg/common/config/config.go

@@ -71,15 +71,39 @@ type Minio struct {
 }
 
 type Mongo struct {
-	URI         string   `yaml:"uri"`
-	Address     []string `yaml:"address"`
-	Database    string   `yaml:"database"`
-	Username    string   `yaml:"username"`
-	Password    string   `yaml:"password"`
-	AuthSource  string   `yaml:"authSource"`
-	MaxPoolSize int      `yaml:"maxPoolSize"`
-	MaxRetry    int      `yaml:"maxRetry"`
+	URI            string   `yaml:"uri"`
+	Address        []string `yaml:"address"`
+	Database       string   `yaml:"database"`
+	Username       string   `yaml:"username"`
+	Password       string   `yaml:"password"`
+	AuthSource     string   `yaml:"authSource"`
+	MaxPoolSize    int      `yaml:"maxPoolSize"`
+	MaxRetry       int      `yaml:"maxRetry"`
+	MongoMode      string   `yaml:"mongoMode"`
+	ReplicaSet     ReplicaSetConfig
+	ReadPreference ReadPrefConfig
+	WriteConcern   WriteConcernConfig
 }
+
+type ReplicaSetConfig struct {
+	Name         string        `yaml:"name"`
+	Hosts        []string      `yaml:"hosts"`
+	ReadConcern  string        `yaml:"readConcern"`
+	MaxStaleness time.Duration `yaml:"maxStaleness"`
+}
+
+type ReadPrefConfig struct {
+	Mode         string              `yaml:"mode"`
+	TagSets      []map[string]string `yaml:"tagSets"`
+	MaxStaleness time.Duration       `yaml:"maxStaleness"`
+}
+
+type WriteConcernConfig struct {
+	W        any           `yaml:"w"`
+	J        bool          `yaml:"j"`
+	WTimeout time.Duration `yaml:"wtimeout"`
+}
+
 type Kafka struct {
 	Username           string   `yaml:"username"`
 	Password           string   `yaml:"password"`
@@ -323,14 +347,22 @@ type RPC struct {
 }
 
 type Redis struct {
-	Disable     bool     `yaml:"-"`
-	Address     []string `yaml:"address"`
-	Username    string   `yaml:"username"`
-	Password    string   `yaml:"password"`
-	ClusterMode bool     `yaml:"clusterMode"`
-	DB          int      `yaml:"storage"`
-	MaxRetry    int      `yaml:"maxRetry"`
-	PoolSize    int      `yaml:"poolSize"`
+	Disable      bool     `yaml:"-"`
+	Address      []string `yaml:"address"`
+	Username     string   `yaml:"username"`
+	Password     string   `yaml:"password"`
+	RedisMode    string   `yaml:"redisMode"`
+	DB           int      `yaml:"db"`
+	MaxRetry     int      `yaml:"maxRetry"`
+	PoolSize     int      `yaml:"poolSize"`
+	SentinelMode Sentinel `yaml:"sentinelMode"`
+}
+
+type Sentinel struct {
+	MasterName     string   `yaml:"masterName"`
+	SentinelAddrs  []string `yaml:"sentinelsAddrs"`
+	RouteByLatency bool     `yaml:"routeByLatency"`
+	RouteRandomly  bool     `yaml:"routeRandomly"`
 }
 
 type BeforeConfig struct {
@@ -348,8 +380,11 @@ type AfterConfig struct {
 }
 
 type Share struct {
-	Secret         string         `yaml:"secret"`
-	IMAdminUserID  []string       `yaml:"imAdminUserID"`
+	Secret      string `yaml:"secret"`
+	IMAdminUser struct {
+		UserIDs   []string `yaml:"userIDs"`
+		Nicknames []string `yaml:"nicknames"`
+	} `yaml:"imAdminUser"`
 	MultiLogin     MultiLogin     `yaml:"multiLogin"`
 	RPCMaxBodySize MaxRequestBody `yaml:"rpcMaxBodySize"`
 }
@@ -482,18 +517,41 @@ func (m *Mongo) Build() *mongoutil.Config {
 		AuthSource:  m.AuthSource,
 		MaxPoolSize: m.MaxPoolSize,
 		MaxRetry:    m.MaxRetry,
+		MongoMode:   m.MongoMode,
+		ReplicaSet: &mongoutil.ReplicaSetConfig{
+			Name:         m.ReplicaSet.Name,
+			Hosts:        m.ReplicaSet.Hosts,
+			ReadConcern:  m.ReplicaSet.ReadConcern,
+			MaxStaleness: m.ReplicaSet.MaxStaleness,
+		},
+		ReadPreference: &mongoutil.ReadPrefConfig{
+			Mode:         m.ReadPreference.Mode,
+			TagSets:      m.ReadPreference.TagSets,
+			MaxStaleness: m.ReadPreference.MaxStaleness,
+		},
+		WriteConcern: &mongoutil.WriteConcernConfig{
+			W:        m.WriteConcern.W,
+			J:        m.WriteConcern.J,
+			WTimeout: m.WriteConcern.WTimeout,
+		},
 	}
 }
 
 func (r *Redis) Build() *redisutil.Config {
 	return &redisutil.Config{
-		ClusterMode: r.ClusterMode,
-		Address:     r.Address,
-		Username:    r.Username,
-		Password:    r.Password,
-		DB:          r.DB,
-		MaxRetry:    r.MaxRetry,
-		PoolSize:    r.PoolSize,
+		RedisMode: r.RedisMode,
+		Address:   r.Address,
+		Username:  r.Username,
+		Password:  r.Password,
+		DB:        r.DB,
+		MaxRetry:  r.MaxRetry,
+		PoolSize:  r.PoolSize,
+		Sentinel: &redisutil.Sentinel{
+			MasterName:     r.SentinelMode.MasterName,
+			SentinelAddrs:  r.SentinelMode.SentinelAddrs,
+			RouteByLatency: r.SentinelMode.RouteByLatency,
+			RouteRandomly:  r.SentinelMode.RouteRandomly,
+		},
 	}
 }
 

pkg/common/discovery/discoveryregister.go

@@ -35,7 +35,7 @@ func NewDiscoveryRegister(discovery *config.Discovery, watchNames []string) (dis
 		return standalone.GetSvcDiscoveryRegistry(), nil
 	}
 	if runtimeenv.RuntimeEnvironment() == config.KUBERNETES {
-		return kubernetes.NewKubernetesConnManager(discovery.Kubernetes.Namespace,
+		return kubernetes.NewConnManager(discovery.Kubernetes.Namespace, nil,
 			grpc.WithDefaultCallOptions(
 				grpc.MaxCallSendMsgSize(1024*1024*20),
 			),

pkg/common/prommetrics/prommetrics.go

@@ -85,6 +85,8 @@ func Start(listener net.Listener) error {
 const (
 	APIKeyName             = "api"
 	MessageTransferKeyName = "message-transfer"
+
+	TTL = 300
 )
 
 type Target struct {
@@ -97,10 +99,14 @@ type RespTarget struct {
 	Labels  map[string]string `json:"labels"`
 }
 
-func BuildDiscoveryKey(name string) string {
+func BuildDiscoveryKeyPrefix(name string) string {
 	return fmt.Sprintf("%s/%s/%s", "openim", "prometheus_discovery", name)
 }
 
+func BuildDiscoveryKey(name string, index int) string {
+	return fmt.Sprintf("%s/%s/%s/%d", "openim", "prometheus_discovery", name, index)
+}
+
 func BuildDefaultTarget(host string, ip int) Target {
 	return Target{
 		Target: fmt.Sprintf("%s:%d", host, ip),

pkg/common/startrpc/start.go

@@ -50,7 +50,7 @@ func init() {
 func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *conf.Prometheus, listenIP,
 	registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, notification *conf.Notification, config T,
 	watchConfigNames []string, watchServiceNames []string,
-	rpcFn func(ctx context.Context, config T, client discovery.Conn, server grpc.ServiceRegistrar) error,
+	rpcFn func(ctx context.Context, config T, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error,
 	options ...grpc.ServerOption) error {
 
 	if notification != nil {
@@ -69,8 +69,8 @@ func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *c
 		grpcsrv.GrpcServerRequestValidate(),
 		grpcsrv.GrpcServerPanicCapture(),
 	)
-	if shareConfig != nil && len(shareConfig.IMAdminUserID) > 0 {
-		options = append(options, grpcServerIMAdminUserID(shareConfig.IMAdminUserID))
+	if shareConfig != nil && len(shareConfig.IMAdminUser.UserIDs) > 0 {
+		options = append(options, grpcServerIMAdminUserID(shareConfig.IMAdminUser.UserIDs))
 	}
 	var clientOptions []grpc.DialOption
 	if maxRequestBody != nil {
@@ -148,9 +148,11 @@ func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *c
 		if err != nil {
 			return err
 		}
-		if err := client.SetKey(ctx, prommetrics.BuildDiscoveryKey(prommetrics.APIKeyName), target); err != nil {
-			if !errors.Is(err, discovery.ErrNotSupportedKeyValue) {
-				return err
+		if autoSetPorts {
+			if err = client.SetWithLease(ctx, prommetrics.BuildDiscoveryKey(rpcRegisterName, index), target, prommetrics.TTL); err != nil {
+				if !errors.Is(err, discovery.ErrNotSupported) {
+					return err
+				}
 			}
 		}
 		go func() {

pkg/common/storage/cache/redis/online_test.go

@@ -26,7 +26,7 @@ func TestName111111(t *testing.T) {
 			"172.16.8.124:7005",
 			"172.16.8.124:7006",
 		},
-		ClusterMode: true,
+		RedisMode: "cluster",
 		Password:    "passwd123",
 		//Address:  []string{"localhost:16379"},
 		//Password: "openIM123",

pkg/common/storage/controller/msg.go

@@ -671,12 +671,18 @@ func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.Searc
 func (db *commonMsgDatabase) FindOneByDocIDs(ctx context.Context, conversationIDs []string, seqs map[string]int64) (map[string]*sdkws.MsgData, error) {
 	totalMsgs := make(map[string]*sdkws.MsgData)
 	for _, conversationID := range conversationIDs {
-		seq := seqs[conversationID]
+		seq, ok := seqs[conversationID]
+		if !ok {
+			log.ZWarn(ctx, "seq not found for conversationID", errs.New("seq not found for conversation"), "conversationID", conversationID)
+			continue
+		}
 		docID := db.msgTable.GetDocID(conversationID, seq)
 		msgs, err := db.msgDocDatabase.FindOneByDocID(ctx, docID)
 		if err != nil {
-			return nil, err
+			log.ZWarn(ctx, "FindOneByDocID failed", err, "conversationID", conversationID, "docID", docID, "seq", seq)
+			continue
 		}
+
 		index := db.msgTable.GetMsgIndex(seq)
 		totalMsgs[conversationID] = convert.MsgDB2Pb(msgs.Msg[index].Msg)
 	}

pkg/common/storage/controller/user.go

@@ -97,16 +97,34 @@ func (u *userDatabase) InitOnce(ctx context.Context, users []*model.User) error
 	}
 
 	// Determine which users are missing from the database.
-	missingUsers := datautil.SliceAnySub(users, existingUsers, func(e *model.User) string {
+	var (
+		missing, update []*model.User
+	)
+	existMap := datautil.SliceToMap(existingUsers, func(e *model.User) string {
 		return e.UserID
 	})
+	orgMap := datautil.SliceToMap(users, func(e *model.User) string { return e.UserID })
+	for k, u1 := range orgMap {
+		if u2, ok := existMap[k]; !ok {
+			missing = append(missing, u1)
+		} else if u1.Nickname != u2.Nickname {
+			update = append(update, u1)
+		}
+	}
 
 	// Create records for missing users.
-	if len(missingUsers) > 0 {
-		if err := u.userDB.Create(ctx, missingUsers); err != nil {
+	if len(missing) > 0 {
+		if err := u.userDB.Create(ctx, missing); err != nil {
 			return err
 		}
 	}
+	if len(update) > 0 {
+		for i := range update {
+			if err := u.userDB.UpdateByMap(ctx, update[i].UserID, map[string]any{"nickname": update[i].Nickname}); err != nil {
+				return err
+			}
+		}
+	}
 
 	return nil
 }

pkg/common/storage/database/mgo/cache.go

@@ -127,7 +127,7 @@ func (x *CacheMgo) Del(ctx context.Context, key []string) error {
 		return nil
 	}
 	_, err := x.coll.DeleteMany(ctx, bson.M{"key": bson.M{"$in": key}})
-	return err
+	return errs.Wrap(err)
 }
 
 func (x *CacheMgo) lockKey(key string) string {

pkg/common/storage/database/mgo/msg.go

@@ -321,7 +321,12 @@ func (m *MsgMgo) searchMessageIndex(ctx context.Context, filter any, nextID prim
 }
 
 func (m *MsgMgo) searchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []searchMessageIndex, error) {
-	filter := bson.M{}
+	filter := bson.M{
+		"msgs.msg": bson.M{
+			"$exists": true,
+			"$type":   "object",
+		},
+	}
 	if req.RecvID != "" {
 		filter["$or"] = bson.A{
 			bson.M{"msgs.msg.recv_id": req.RecvID},

pkg/common/storage/database/mgo/msg_test.go

@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
+	"github.com/openimsdk/protocol/msg"
+	"github.com/openimsdk/protocol/sdkws"
 	"github.com/openimsdk/tools/db/mongoutil"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -148,3 +150,29 @@ func TestName5(t *testing.T) {
 //	}
 //	t.Log(seq, sendTime)
 //}
+
+func TestSearchMessage(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
+	defer cancel()
+	cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.135:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)))
+
+	msgMongo, err := NewMsgMongo(cli.Database("openim_v3"))
+	if err != nil {
+		panic(err)
+	}
+	ts := time.Now().Add(-time.Hour * 24 * 5).UnixMilli()
+	t.Log(ts)
+	req := &msg.SearchMessageReq{
+		//SendID: "yjz",
+		//RecvID: "aibot",
+		Pagination: &sdkws.RequestPagination{
+			PageNumber: 1,
+			ShowNumber: 20,
+		},
+	}
+	count, resp, err := msgMongo.SearchMessage(ctx, req)
+	if err != nil {
+		panic(err)
+	}
+	t.Log(resp, count)
+}

pkg/common/storage/model/msg.go

@@ -120,15 +120,11 @@ func (m *MsgDocModel) GetDocID(conversationID string, seq int64) string {
 
 func (m *MsgDocModel) GetDocIDSeqsMap(conversationID string, seqs []int64) map[string][]int64 {
 	t := make(map[string][]int64)
-	for i := 0; i < len(seqs); i++ {
-		docID := m.GetDocID(conversationID, seqs[i])
-		if value, ok := t[docID]; !ok {
-			var temp []int64
-			t[docID] = append(temp, seqs[i])
-		} else {
-			t[docID] = append(value, seqs[i])
-		}
+	for _, seq := range seqs {
+		docID := m.GetDocID(conversationID, seq)
+		t[docID] = append(t[docID], seq)
 	}
+
 	return t
 }