feat: optimize code and support running in single process mode (#3142)

* pb

* fix: Modifying other fields while setting IsPrivateChat does not take effect

* fix: quote message error revoke

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* refactoring scheduled tasks

* upgrading pkg tools

* fix

* fix

* optimize log output

* feat: support GetLastMessage

* feat: support GetLastMessage

* feat: s3 switch

* feat: s3 switch

* fix: GetUsersOnline

* feat: SendBusinessNotification supported configuration parameters

* feat: SendBusinessNotification supported configuration parameters

* feat: SendBusinessNotification supported configuration parameters

* feat: seq conversion failed without exiting

* monolithic

* fix: DeleteDoc crash

* fix: DeleteDoc crash

* fix: monolithic

* fix: monolithic

* fix: fill send time

* fix: fill send time

* fix: crash caused by withdrawing messages from users who have left the group

* fix: mq

* fix: mq

* fix: user msg timestamp

* fix: mq

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* seq read config

* seq read config

* 1

* 1

* fix: the source message of the reference is withdrawn, and the referenced message is deleted

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

* 1

internal/push/callback.go

@@ -17,6 +17,7 @@ package push
 import (
 	"context"
 	"encoding/json"
+
 	"github.com/openimsdk/open-im-server/v3/pkg/common/webhook"
 
 	"github.com/openimsdk/open-im-server/v3/pkg/callbackstruct"

internal/push/offlinepush_handler.go

@@ -3,7 +3,6 @@ package push
 import (
 	"context"
 
-	"github.com/IBM/sarama"
 	"github.com/openimsdk/open-im-server/v3/internal/push/offlinepush"
 	"github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics"
@@ -12,40 +11,21 @@ import (
 	"github.com/openimsdk/protocol/sdkws"
 	"github.com/openimsdk/tools/errs"
 	"github.com/openimsdk/tools/log"
-	"github.com/openimsdk/tools/mq/kafka"
 	"github.com/openimsdk/tools/utils/jsonutil"
 	"google.golang.org/protobuf/proto"
 )
 
 type OfflinePushConsumerHandler struct {
-	OfflinePushConsumerGroup *kafka.MConsumerGroup
-	offlinePusher            offlinepush.OfflinePusher
+	offlinePusher offlinepush.OfflinePusher
 }
 
-func NewOfflinePushConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher) (*OfflinePushConsumerHandler, error) {
-	var offlinePushConsumerHandler OfflinePushConsumerHandler
-	var err error
-	offlinePushConsumerHandler.offlinePusher = offlinePusher
-	offlinePushConsumerHandler.OfflinePushConsumerGroup, err = kafka.NewMConsumerGroup(config.KafkaConfig.Build(), config.KafkaConfig.ToOfflineGroupID,
-		[]string{config.KafkaConfig.ToOfflinePushTopic}, true)
-	if err != nil {
-		return nil, err
+func NewOfflinePushConsumerHandler(offlinePusher offlinepush.OfflinePusher) *OfflinePushConsumerHandler {
+	return &OfflinePushConsumerHandler{
+		offlinePusher: offlinePusher,
 	}
-	return &offlinePushConsumerHandler, nil
 }
 
-func (*OfflinePushConsumerHandler) Setup(sarama.ConsumerGroupSession) error   { return nil }
-func (*OfflinePushConsumerHandler) Cleanup(sarama.ConsumerGroupSession) error { return nil }
-func (o *OfflinePushConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
-	for msg := range claim.Messages() {
-		ctx := o.OfflinePushConsumerGroup.GetContextFromMsg(msg)
-		o.handleMsg2OfflinePush(ctx, msg.Value)
-		sess.MarkMessage(msg, "")
-	}
-	return nil
-}
-
-func (o *OfflinePushConsumerHandler) handleMsg2OfflinePush(ctx context.Context, msg []byte) {
+func (o *OfflinePushConsumerHandler) HandleMsg2OfflinePush(ctx context.Context, msg []byte) {
 	offlinePushMsg := pbpush.PushMsgReq{}
 	if err := proto.Unmarshal(msg, &offlinePushMsg); err != nil {
 		log.ZError(ctx, "offline push Unmarshal msg err", err, "msg", string(msg))

internal/push/onlinepusher.go

@@ -2,7 +2,7 @@ package push
 
 import (
 	"context"
-	"errors"
+	"fmt"
 	"sync"
 
 	"github.com/openimsdk/protocol/msggateway"
@@ -11,6 +11,7 @@ import (
 	"github.com/openimsdk/tools/errs"
 	"github.com/openimsdk/tools/log"
 	"github.com/openimsdk/tools/utils/datautil"
+	"github.com/openimsdk/tools/utils/runtimeenv"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 
@@ -30,37 +31,36 @@ func newEmptyOnlinePusher() *emptyOnlinePusher {
 	return &emptyOnlinePusher{}
 }
 
-func (emptyOnlinePusher) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData,
-	pushToUserIDs []string) (wsResults []*msggateway.SingleMsgToUserResults, err error) {
+func (emptyOnlinePusher) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) (wsResults []*msggateway.SingleMsgToUserResults, err error) {
 	log.ZInfo(ctx, "emptyOnlinePusher GetConnsAndOnlinePush", nil)
 	return nil, nil
 }
-func (u emptyOnlinePusher) GetOnlinePushFailedUserIDs(ctx context.Context, msg *sdkws.MsgData,
-	wsResults []*msggateway.SingleMsgToUserResults, pushToUserIDs *[]string) []string {
+func (u emptyOnlinePusher) GetOnlinePushFailedUserIDs(ctx context.Context, msg *sdkws.MsgData, wsResults []*msggateway.SingleMsgToUserResults, pushToUserIDs *[]string) []string {
 	log.ZInfo(ctx, "emptyOnlinePusher GetOnlinePushFailedUserIDs", nil)
 	return nil
 }
 
-func NewOnlinePusher(disCov discovery.SvcDiscoveryRegistry, config *Config) OnlinePusher {
-
-	if config.runTimeEnv == conf.KUBERNETES {
-		return NewDefaultAllNode(disCov, config)
+func NewOnlinePusher(disCov discovery.Conn, config *Config) (OnlinePusher, error) {
+	if conf.Standalone() {
+		return NewDefaultAllNode(disCov, config), nil
+	}
+	if runtimeenv.RuntimeEnvironment() == conf.KUBERNETES {
+		return NewDefaultAllNode(disCov, config), nil
 	}
 	switch config.Discovery.Enable {
 	case conf.ETCD:
-		return NewDefaultAllNode(disCov, config)
+		return NewDefaultAllNode(disCov, config), nil
 	default:
-		log.ZError(context.Background(), "NewOnlinePusher is error", errs.Wrap(errors.New("unsupported discovery type")), "type", config.Discovery.Enable)
-		return nil
+		return nil, errs.New(fmt.Sprintf("unsupported discovery type %s", config.Discovery.Enable))
 	}
 }
 
 type DefaultAllNode struct {
-	disCov discovery.SvcDiscoveryRegistry
+	disCov discovery.Conn
 	config *Config
 }
 
-func NewDefaultAllNode(disCov discovery.SvcDiscoveryRegistry, config *Config) *DefaultAllNode {
+func NewDefaultAllNode(disCov discovery.Conn, config *Config) *DefaultAllNode {
 	return &DefaultAllNode{disCov: disCov, config: config}
 }
 
@@ -166,7 +166,7 @@ func (k *K8sStaticConsistentHash) GetConnsAndOnlinePush(ctx context.Context, msg
 		}
 	}
 	log.ZDebug(ctx, "genUsers send hosts struct:", "usersHost", usersHost)
-	var usersConns = make(map[*grpc.ClientConn][]string)
+	var usersConns = make(map[grpc.ClientConnInterface][]string)
 	for host, userIds := range usersHost {
 		tconn, _ := k.disCov.GetConn(ctx, host)
 		usersConns[tconn] = userIds

internal/push/push.go

@@ -2,39 +2,43 @@ package push
 
 import (
 	"context"
+	"math/rand"
+	"strconv"
 
 	"github.com/openimsdk/open-im-server/v3/internal/push/offlinepush"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/mcache"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo"
+	"github.com/openimsdk/open-im-server/v3/pkg/dbbuild"
+	"github.com/openimsdk/open-im-server/v3/pkg/mqbuild"
 	pbpush "github.com/openimsdk/protocol/push"
-	"github.com/openimsdk/tools/db/redisutil"
 	"github.com/openimsdk/tools/discovery"
-	"github.com/openimsdk/tools/utils/runtimeenv"
+	"github.com/openimsdk/tools/log"
+	"github.com/openimsdk/tools/mcontext"
 	"google.golang.org/grpc"
 )
 
 type pushServer struct {
 	pbpush.UnimplementedPushMsgServiceServer
 	database      controller.PushDatabase
-	disCov        discovery.SvcDiscoveryRegistry
+	disCov        discovery.Conn
 	offlinePusher offlinepush.OfflinePusher
-	pushCh        *ConsumerHandler
-	offlinePushCh *OfflinePushConsumerHandler
 }
 
 type Config struct {
 	RpcConfig          config.Push
 	RedisConfig        config.Redis
+	MongoConfig        config.Mongo
 	KafkaConfig        config.Kafka
 	NotificationConfig config.Notification
 	Share              config.Share
 	WebhooksConfig     config.Webhooks
 	LocalCacheConfig   config.LocalCache
 	Discovery          config.Discovery
-	FcmConfigPath      string
-
-	runTimeEnv string
+	FcmConfigPath      config.Path
 }
 
 func (p pushServer) DelUserPushToken(ctx context.Context,
@@ -45,42 +49,90 @@ func (p pushServer) DelUserPushToken(ctx context.Context,
 	return &pbpush.DelUserPushTokenResp{}, nil
 }
 
-func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error {
-	config.runTimeEnv = runtimeenv.PrintRuntimeEnvironment()
-
-	rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build())
+func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error {
+	dbb := dbbuild.NewBuilder(&config.MongoConfig, &config.RedisConfig)
+	rdb, err := dbb.Redis(ctx)
 	if err != nil {
 		return err
 	}
-	cacheModel := redis.NewThirdCache(rdb)
-	offlinePusher, err := offlinepush.NewOfflinePusher(&config.RpcConfig, cacheModel, config.FcmConfigPath)
+	var cacheModel cache.ThirdCache
+	if rdb == nil {
+		mdb, err := dbb.Mongo(ctx)
+		if err != nil {
+			return err
+		}
+		mc, err := mgo.NewCacheMgo(mdb.GetDB())
+		if err != nil {
+			return err
+		}
+		cacheModel = mcache.NewThirdCache(mc)
+	} else {
+		cacheModel = redis.NewThirdCache(rdb)
+	}
+	offlinePusher, err := offlinepush.NewOfflinePusher(&config.RpcConfig, cacheModel, string(config.FcmConfigPath))
+	if err != nil {
+		return err
+	}
+	builder := mqbuild.NewBuilder(&config.KafkaConfig)
+
+	offlinePushProducer, err := builder.GetTopicProducer(ctx, config.KafkaConfig.ToOfflinePushTopic)
+	if err != nil {
+		return err
+	}
+	database := controller.NewPushDatabase(cacheModel, offlinePushProducer)
+
+	pushConsumer, err := builder.GetTopicConsumer(ctx, config.KafkaConfig.ToPushTopic)
+	if err != nil {
+		return err
+	}
+	offlinePushConsumer, err := builder.GetTopicConsumer(ctx, config.KafkaConfig.ToOfflinePushTopic)
 	if err != nil {
 		return err
 	}
 
-	database := controller.NewPushDatabase(cacheModel, &config.KafkaConfig)
-
-	consumer, err := NewConsumerHandler(ctx, config, database, offlinePusher, rdb, client)
+	pushHandler, err := NewConsumerHandler(ctx, config, database, offlinePusher, rdb, client)
 	if err != nil {
 		return err
 	}
 
-	offlinePushConsumer, err := NewOfflinePushConsumerHandler(config, offlinePusher)
-	if err != nil {
-		return err
-	}
+	offlineHandler := NewOfflinePushConsumerHandler(offlinePusher)
 
 	pbpush.RegisterPushMsgServiceServer(server, &pushServer{
 		database:      database,
 		disCov:        client,
 		offlinePusher: offlinePusher,
-		pushCh:        consumer,
-		offlinePushCh: offlinePushConsumer,
 	})
 
-	go consumer.pushConsumerGroup.RegisterHandleAndConsumer(ctx, consumer)
+	go func() {
+		pushHandler.WaitCache()
+		fn := func(ctx context.Context, key string, value []byte) error {
+			pushHandler.HandleMs2PsChat(ctx, value)
+			return nil
+		}
+		consumerCtx := mcontext.SetOperationID(context.Background(), "push_"+strconv.Itoa(int(rand.Uint32())))
+		log.ZInfo(consumerCtx, "begin consume messages")
+		for {
+			if err := pushConsumer.Subscribe(consumerCtx, fn); err != nil {
+				log.ZError(consumerCtx, "subscribe err", err)
+				return
+			}
+		}
+	}()
 
-	go offlinePushConsumer.OfflinePushConsumerGroup.RegisterHandleAndConsumer(ctx, offlinePushConsumer)
+	go func() {
+		fn := func(ctx context.Context, key string, value []byte) error {
+			offlineHandler.HandleMsg2OfflinePush(ctx, value)
+			return nil
+		}
+		consumerCtx := mcontext.SetOperationID(context.Background(), "push_"+strconv.Itoa(int(rand.Uint32())))
+		log.ZInfo(consumerCtx, "begin consume messages")
+		for {
+			if err := offlinePushConsumer.Subscribe(consumerCtx, fn); err != nil {
+				log.ZError(consumerCtx, "subscribe err", err)
+				return
+			}
+		}
+	}()
 
 	return nil
 }

internal/push/push_handler.go

@@ -3,13 +3,8 @@ package push
 import (
 	"context"
 	"encoding/json"
-	"math/rand"
-	"strconv"
 	"time"
 
-	"github.com/openimsdk/open-im-server/v3/pkg/rpcli"
-
-	"github.com/IBM/sarama"
 	"github.com/openimsdk/open-im-server/v3/internal/push/offlinepush"
 	"github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics"
@@ -17,6 +12,7 @@ import (
 	"github.com/openimsdk/open-im-server/v3/pkg/common/webhook"
 	"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"
 	"github.com/openimsdk/open-im-server/v3/pkg/rpccache"
+	"github.com/openimsdk/open-im-server/v3/pkg/rpcli"
 	"github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil"
 	"github.com/openimsdk/protocol/constant"
 	"github.com/openimsdk/protocol/msggateway"
@@ -25,7 +21,6 @@ import (
 	"github.com/openimsdk/tools/discovery"
 	"github.com/openimsdk/tools/log"
 	"github.com/openimsdk/tools/mcontext"
-	"github.com/openimsdk/tools/mq/kafka"
 	"github.com/openimsdk/tools/utils/datautil"
 	"github.com/openimsdk/tools/utils/jsonutil"
 	"github.com/openimsdk/tools/utils/timeutil"
@@ -34,7 +29,7 @@ import (
 )
 
 type ConsumerHandler struct {
-	pushConsumerGroup      *kafka.MConsumerGroup
+	//pushConsumerGroup      mq.Consumer
 	offlinePusher          offlinepush.OfflinePusher
 	onlinePusher           OnlinePusher
 	pushDatabase           controller.PushDatabase
@@ -49,15 +44,7 @@ type ConsumerHandler struct {
 	conversationClient     *rpcli.ConversationClient
 }
 
-func NewConsumerHandler(ctx context.Context, config *Config, database controller.PushDatabase, offlinePusher offlinepush.OfflinePusher, rdb redis.UniversalClient,
-	client discovery.SvcDiscoveryRegistry) (*ConsumerHandler, error) {
-	var consumerHandler ConsumerHandler
-	var err error
-	consumerHandler.pushConsumerGroup, err = kafka.NewMConsumerGroup(config.KafkaConfig.Build(), config.KafkaConfig.ToPushGroupID,
-		[]string{config.KafkaConfig.ToPushTopic}, true)
-	if err != nil {
-		return nil, err
-	}
+func NewConsumerHandler(ctx context.Context, config *Config, database controller.PushDatabase, offlinePusher offlinepush.OfflinePusher, rdb redis.UniversalClient, client discovery.Conn) (*ConsumerHandler, error) {
 	userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User)
 	if err != nil {
 		return nil, err
@@ -74,13 +61,18 @@ func NewConsumerHandler(ctx context.Context, config *Config, database controller
 	if err != nil {
 		return nil, err
 	}
+	onlinePusher, err := NewOnlinePusher(client, config)
+	if err != nil {
+		return nil, err
+	}
+	var consumerHandler ConsumerHandler
 	consumerHandler.userClient = rpcli.NewUserClient(userConn)
 	consumerHandler.groupClient = rpcli.NewGroupClient(groupConn)
 	consumerHandler.msgClient = rpcli.NewMsgClient(msgConn)
 	consumerHandler.conversationClient = rpcli.NewConversationClient(conversationConn)
 
 	consumerHandler.offlinePusher = offlinePusher
-	consumerHandler.onlinePusher = NewOnlinePusher(client, config)
+	consumerHandler.onlinePusher = onlinePusher
 	consumerHandler.groupLocalCache = rpccache.NewGroupLocalCache(consumerHandler.groupClient, &config.LocalCacheConfig, rdb)
 	consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationClient, &config.LocalCacheConfig, rdb)
 	consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL)
@@ -93,7 +85,7 @@ func NewConsumerHandler(ctx context.Context, config *Config, database controller
 	return &consumerHandler, nil
 }
 
-func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) {
+func (c *ConsumerHandler) HandleMs2PsChat(ctx context.Context, msg []byte) {
 	msgFromMQ := pbpush.PushMsgReq{}
 	if err := proto.Unmarshal(msg, &msgFromMQ); err != nil {
 		log.ZError(ctx, "push Unmarshal msg err", err, "msg", string(msg))
@@ -127,25 +119,12 @@ func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) {
 	}
 }
 
-func (*ConsumerHandler) Setup(sarama.ConsumerGroupSession) error { return nil }
-
-func (*ConsumerHandler) Cleanup(sarama.ConsumerGroupSession) error { return nil }
-
-func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
+func (c *ConsumerHandler) WaitCache() {
 	c.onlineCache.Lock.Lock()
 	for c.onlineCache.CurrentPhase.Load() < rpccache.DoSubscribeOver {
 		c.onlineCache.Cond.Wait()
 	}
 	c.onlineCache.Lock.Unlock()
-	ctx := mcontext.SetOperationID(context.TODO(), strconv.FormatInt(time.Now().UnixNano()+int64(rand.Uint32()), 10))
-	log.ZInfo(ctx, "begin consume messages")
-
-	for msg := range claim.Messages() {
-		ctx := c.pushConsumerGroup.GetContextFromMsg(msg)
-		c.handleMs2PsChat(ctx, msg.Value)
-		sess.MarkMessage(msg, "")
-	}
-	return nil
 }
 
 // Push2User Suitable for two types of conversations, one is SingleChatType and the other is NotificationChatType.