feat: optimize code and support running in single process mode (#3142)

* pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * monolithic * fix: DeleteDoc crash * fix: DeleteDoc crash * fix: monolithic * fix: monolithic * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: mq * fix: mq * fix: user msg timestamp * fix: mq * 1 * 1 * 1 * 1 * 1 * 1 * 1 * seq read config * seq read config * 1 * 1 * fix: the source message of the reference is withdrawn, and the referenced message is deleted * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1
2026-04-28 06:19:20 +08:00 · 2025-02-14 16:18:27 +08:00
parent e37ea50b94
commit 9ed6200e45
105 changed files with 3385 additions and 1925 deletions
@@ -22,7 +22,6 @@ import (

 	"github.com/openimsdk/open-im-server/v3/pkg/authverify"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/servererrs"
-	"github.com/openimsdk/open-im-server/v3/pkg/common/startrpc"
 	"github.com/openimsdk/protocol/constant"
 	"github.com/openimsdk/protocol/msggateway"
 	"github.com/openimsdk/protocol/sdkws"
@@ -35,7 +34,7 @@ import (
 	"google.golang.org/grpc"
 )

-func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error {
+func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.Conn, server grpc.ServiceRegistrar) error {
 	userConn, err := disCov.GetConn(ctx, config.Discovery.RpcService.User)
 	if err != nil {
 		return err
@@ -51,26 +50,26 @@ func (s *Server) InitServer(ctx context.Context, config *Config, disCov discover
 	return nil
 }

-func (s *Server) Start(ctx context.Context, index int, conf *Config) error {
-	return startrpc.Start(ctx, &conf.Discovery, &conf.MsgGateway.Prometheus, conf.MsgGateway.ListenIP,
-		conf.MsgGateway.RPC.RegisterIP,
-		conf.MsgGateway.RPC.AutoSetPorts, conf.MsgGateway.RPC.Ports, index,
-		conf.Discovery.RpcService.MessageGateway,
-		nil,
-		conf,
-		[]string{
-			conf.Share.GetConfigFileName(),
-			conf.Discovery.GetConfigFileName(),
-			conf.MsgGateway.GetConfigFileName(),
-			conf.WebhooksConfig.GetConfigFileName(),
-			conf.RedisConfig.GetConfigFileName(),
-		},
-		[]string{
-			conf.Discovery.RpcService.MessageGateway,
-		},
-		s.InitServer,
-	)
-}
+//func (s *Server) Start(ctx context.Context, index int, conf *Config) error {
+//	return startrpc.Start(ctx, &conf.Discovery, &conf.MsgGateway.Prometheus, conf.MsgGateway.ListenIP,
+//		conf.MsgGateway.RPC.RegisterIP,
+//		conf.MsgGateway.RPC.AutoSetPorts, conf.MsgGateway.RPC.Ports, index,
+//		conf.Discovery.RpcService.MessageGateway,
+//		nil,
+//		conf,
+//		[]string{
+//			conf.Share.GetConfigFileName(),
+//			conf.Discovery.GetConfigFileName(),
+//			conf.MsgGateway.GetConfigFileName(),
+//			conf.WebhooksConfig.GetConfigFileName(),
+//			conf.RedisConfig.GetConfigFileName(),
+//		},
+//		[]string{
+//			conf.Discovery.RpcService.MessageGateway,
+//		},
+//		s.InitServer,
+//	)
+//}

 type Server struct {
 	msggateway.UnimplementedMsgGatewayServer
@@ -19,10 +19,12 @@ import (
 	"time"

 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	"github.com/openimsdk/open-im-server/v3/pkg/dbbuild"
 	"github.com/openimsdk/open-im-server/v3/pkg/rpccache"
-	"github.com/openimsdk/tools/db/redisutil"
+	"github.com/openimsdk/tools/discovery"
 	"github.com/openimsdk/tools/utils/datautil"
 	"github.com/openimsdk/tools/utils/runtimeenv"
+	"google.golang.org/grpc"

 	"github.com/openimsdk/tools/log"
 )
@@ -33,26 +35,25 @@ type Config struct {
 	RedisConfig    config.Redis
 	WebhooksConfig config.Webhooks
 	Discovery      config.Discovery
-
-	RuntimeEnv string
+	Index          config.Index
 }

 // Start run ws server.
-func Start(ctx context.Context, index int, conf *Config) error {
-	conf.RuntimeEnv = runtimeenv.PrintRuntimeEnvironment()
-
-	log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", conf.RuntimeEnv,
+func Start(ctx context.Context, conf *Config, client discovery.Conn, server grpc.ServiceRegistrar) error {
+	log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", runtimeenv.RuntimeEnvironment(),
 		"rpcPorts", conf.MsgGateway.RPC.Ports,
 		"wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports)
-	wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, index)
+	wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, int(conf.Index))
 	if err != nil {
 		return err
 	}

-	rdb, err := redisutil.NewRedisClient(ctx, conf.RedisConfig.Build())
+	dbb := dbbuild.NewBuilder(nil, &conf.RedisConfig)
+	rdb, err := dbb.Redis(ctx)
 	if err != nil {
 		return err
 	}
+
 	longServer := NewWsServer(
 		conf,
 		WithPort(wsPort),
@@ -67,12 +68,50 @@ func Start(ctx context.Context, index int, conf *Config) error {
 		return err
 	})

+	if err := hubServer.InitServer(ctx, conf, client, server); err != nil {
+		return err
+	}
+
 	go longServer.ChangeOnlineStatus(4)

-	netDone := make(chan error)
-	go func() {
-		err = hubServer.Start(ctx, index, conf)
-		netDone <- err
-	}()
-	return hubServer.LongConnServer.Run(netDone)
+	return hubServer.LongConnServer.Run(ctx)
 }
+
+//
+//// Start run ws server.
+//func Start(ctx context.Context, index int, conf *Config) error {
+//	log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", runtimeenv.RuntimeEnvironment(),
+//		"rpcPorts", conf.MsgGateway.RPC.Ports,
+//		"wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports)
+//	wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, index)
+//	if err != nil {
+//		return err
+//	}
+//
+//	rdb, err := redisutil.NewRedisClient(ctx, conf.RedisConfig.Build())
+//	if err != nil {
+//		return err
+//	}
+//	longServer := NewWsServer(
+//		conf,
+//		WithPort(wsPort),
+//		WithMaxConnNum(int64(conf.MsgGateway.LongConnSvr.WebsocketMaxConnNum)),
+//		WithHandshakeTimeout(time.Duration(conf.MsgGateway.LongConnSvr.WebsocketTimeout)*time.Second),
+//		WithMessageMaxMsgLength(conf.MsgGateway.LongConnSvr.WebsocketMaxMsgLen),
+//	)
+//
+//	hubServer := NewServer(longServer, conf, func(srv *Server) error {
+//		var err error
+//		longServer.online, err = rpccache.NewOnlineCache(srv.userClient, nil, rdb, false, longServer.subscriberUserOnlineStatusChanges)
+//		return err
+//	})
+//
+//	go longServer.ChangeOnlineStatus(4)
+//
+//	netDone := make(chan error)
+//	go func() {
+//		err = hubServer.Start(ctx, index, conf)
+//		netDone <- err
+//	}()
+//	return hubServer.LongConnServer.Run(netDone)
+//}
@@ -2,7 +2,6 @@ package msggateway

 import (
 	"context"
-	"errors"
 	"fmt"
 	"net/http"
 	"sync"
@@ -11,7 +10,6 @@ import (

 	"github.com/openimsdk/open-im-server/v3/pkg/rpcli"

-	"github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/webhook"
 	"github.com/openimsdk/open-im-server/v3/pkg/rpccache"
 	pbAuth "github.com/openimsdk/protocol/auth"
@@ -23,19 +21,18 @@ import (
 	"github.com/openimsdk/protocol/constant"
 	"github.com/openimsdk/protocol/msggateway"
 	"github.com/openimsdk/tools/discovery"
-	"github.com/openimsdk/tools/errs"
 	"github.com/openimsdk/tools/log"
 	"github.com/openimsdk/tools/utils/stringutil"
 	"golang.org/x/sync/errgroup"
 )

 type LongConnServer interface {
-	Run(done chan error) error
+	Run(ctx context.Context) error
 	wsHandler(w http.ResponseWriter, r *http.Request)
 	GetUserAllCons(userID string) ([]*Client, bool)
 	GetUserPlatformCons(userID string, platform int) ([]*Client, bool, bool)
 	Validate(s any) error
-	SetDiscoveryRegistry(ctx context.Context, client discovery.SvcDiscoveryRegistry, config *Config) error
+	SetDiscoveryRegistry(ctx context.Context, client discovery.Conn, config *Config) error
 	KickUserConn(client *Client) error
 	UnRegister(c *Client)
 	SetKickHandlerInfo(i *kickHandler)
@@ -60,7 +57,7 @@ type WsServer struct {
 	handshakeTimeout  time.Duration
 	writeBufferSize   int
 	validate          *validator.Validate
-	disCov            discovery.SvcDiscoveryRegistry
+	disCov            discovery.Conn
 	Compressor
 	//Encoder
 	MessageHandler
@@ -75,7 +72,7 @@ type kickHandler struct {
 	newClient  *Client
 }

-func (ws *WsServer) SetDiscoveryRegistry(ctx context.Context, disCov discovery.SvcDiscoveryRegistry, config *Config) error {
+func (ws *WsServer) SetDiscoveryRegistry(ctx context.Context, disCov discovery.Conn, config *Config) error {
 	userConn, err := disCov.GetConn(ctx, config.Discovery.RpcService.User)
 	if err != nil {
 		return err
@@ -158,19 +155,14 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer {
 	}
 }

-func (ws *WsServer) Run(done chan error) error {
-	var (
-		client       *Client
-		netErr       error
-		shutdownDone = make(chan struct{}, 1)
-	)
-
-	server := http.Server{Addr: ":" + stringutil.IntToString(ws.port), Handler: nil}
+func (ws *WsServer) Run(ctx context.Context) error {
+	var client *Client

+	ctx, cancel := context.WithCancelCause(ctx)
 	go func() {
 		for {
 			select {
-			case <-shutdownDone:
+			case <-ctx.Done():
 				return
 			case client = <-ws.registerChan:
 				ws.registerClient(client)
@@ -181,57 +173,56 @@ func (ws *WsServer) Run(done chan error) error {
 			}
 		}
 	}()
-	netDone := make(chan struct{}, 1)
-	go func() {
-		http.HandleFunc("/", ws.wsHandler)
-		err := server.ListenAndServe()
-		if err != nil && !errors.Is(err, http.ErrServerClosed) {
-			netErr = errs.WrapMsg(err, "ws start err", server.Addr)
-			netDone <- struct{}{}
-		}
-	}()
-	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
-	shutDown := func() error {
-		sErr := server.Shutdown(ctx)
-		if sErr != nil {
-			return errs.WrapMsg(sErr, "shutdown err")
-		}
-		close(shutdownDone)
-		return nil
-	}
-	etcd.RegisterShutDown(shutDown)
-	defer cancel()
-	var err error
-	select {
-	case err = <-done:
-		if err := shutDown(); err != nil {
-			return err
-		}
-		if err != nil {
-			return err
-		}
-	case <-netDone:
-	}
-	return netErr

+	done := make(chan struct{})
+	go func() {
+		wsServer := http.Server{Addr: fmt.Sprintf(":%d", ws.port), Handler: nil}
+		http.HandleFunc("/", ws.wsHandler)
+		go func() {
+			defer close(done)
+			<-ctx.Done()
+			_ = wsServer.Shutdown(context.Background())
+		}()
+		err := wsServer.ListenAndServe()
+		if err == nil {
+			err = fmt.Errorf("http server closed")
+		}
+		cancel(fmt.Errorf("msg gateway %w", err))
+	}()
+
+	<-ctx.Done()
+
+	timeout := time.NewTimer(time.Second * 15)
+	defer timeout.Stop()
+	select {
+	case <-timeout.C:
+		log.ZWarn(ctx, "msg gateway graceful stop timeout", nil)
+	case <-done:
+		log.ZDebug(ctx, "msg gateway graceful stop done")
+	}
+	return context.Cause(ctx)
 }

-var concurrentRequest = 3
+const concurrentRequest = 3

 func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *Client) error {
 	conns, err := ws.disCov.GetConns(ctx, ws.msgGatewayConfig.Discovery.RpcService.MessageGateway)
 	if err != nil {
 		return err
 	}
+	if len(conns) == 0 || (len(conns) == 1 && ws.disCov.IsSelfNode(conns[0])) {
+		return nil
+	}
+
 	wg := errgroup.Group{}
 	wg.SetLimit(concurrentRequest)

 	// Online push user online message to other node
 	for _, v := range conns {
 		v := v
-		log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target())
-		if v.Target() == ws.disCov.GetSelfConnTarget() {
-			log.ZDebug(ctx, "Filter out this node", "node", v.Target())
+		log.ZDebug(ctx, "sendUserOnlineInfoToOtherNode conn")
+		if ws.disCov.IsSelfNode(v) {
+			log.ZDebug(ctx, "Filter out this node")
 			continue
 		}

@@ -242,7 +233,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C
 				PlatformID: int32(client.PlatformID), Token: client.token,
 			})
 			if err != nil {
-				log.ZWarn(ctx, "MultiTerminalLoginCheck err", err, "node", v.Target())
+				log.ZWarn(ctx, "MultiTerminalLoginCheck err", err)
 			}
 			return nil
 		})