optimization: change the configuration file from being read globally … (#1935)

* optimization: change the configuration file from being read globally to being read independently. * optimization: change the configuration file from being read globally to being read independently. * optimization: change the configuration file from being read globally to being read independently. * optimization: config file changed to dependency injection. * fix: replace global config with dependency injection * fix: replace global config with dependency injection * fix: import the enough param * fix: import the enough param * fix: import the enough param * fix: fix the component check of path * fix: fix the kafka of tls is nil problem * fix: fix the TLS.CACrt is nil error * fix: fix the valiable shadows problem * fix: fix the comflect * optimization: message remove options. * fix: fix the param pass error * fix: find error * fix: find error * fix: find eror * fix: find error * fix: find error * fix: del the undifined func * fix: find error * fix: fix the error * fix: pass config * fix: find error * fix: find error * fix: find error * fix: find error * fix: find error * fix: fix the config * fix: fix the error * fix: fix the config pass error * fix: fix the eror * fix: fix the error * fix: fix the error * fix: fix the error * fix: find error * fix: fix the error * fix: fix the config * fix: add return err * fix: fix the err2 * fix: err * fix: fix the func * fix: del the chinese comment * fix: fix the func * fix: fix the gateway_test logic * fix: s3 * test * test * fix: not found --------- Co-authored-by: luhaoling <2198702716@qq.com> Co-authored-by: withchao <993506633@qq.com>
2026-04-28 06:19:20 +08:00 · 2024-03-05 17:53:22 +08:00
parent efb8310531
commit 383758782e
114 changed files with 1743 additions and 1802 deletions
@@ -44,21 +44,22 @@ type MsgTransfer struct {
 	// This consumer aggregated messages, subscribed to the topic:ws2ms_chat,
 	// the modification notification is sent to msg_to_modify topic, the message is stored in redis, Incr Redis,
 	// and then the message is sent to ms2pschat topic for push, and the message is sent to msg_to_mongo topic for persistence
-	historyCH *OnlineHistoryRedisConsumerHandler
+	historyCH      *OnlineHistoryRedisConsumerHandler
+	historyMongoCH *OnlineHistoryMongoConsumerHandler
 	// mongoDB batch insert, delete messages in redis after success,
 	// and handle the deletion notification message deleted subscriptions topic: msg_to_mongo
-	historyMongoCH *OnlineHistoryMongoConsumerHandler
-	ctx            context.Context
-	cancel         context.CancelFunc
+	ctx    context.Context
+	cancel context.CancelFunc
+	config *config.GlobalConfig
 }

-func StartTransfer(prometheusPort int) error {
-	rdb, err := cache.NewRedis()
+func StartTransfer(config *config.GlobalConfig, prometheusPort int) error {
+	rdb, err := cache.NewRedis(config)
 	if err != nil {
 		return err
 	}

-	mongo, err := unrelation.NewMongo()
+	mongo, err := unrelation.NewMongo(config)
 	if err != nil {
 		return err
 	}
@@ -66,38 +67,37 @@ func StartTransfer(prometheusPort int) error {
 	if err = mongo.CreateMsgIndex(); err != nil {
 		return err
 	}
-
-	client, err := kdisc.NewDiscoveryRegister(config.Config.Envs.Discovery)
+	client, err := kdisc.NewDiscoveryRegister(config)
 	if err != nil {
 		return err
 	}

-	if err := client.CreateRpcRootNodes(config.Config.GetServiceNames()); err != nil {
+	if err := client.CreateRpcRootNodes(config.GetServiceNames()); err != nil {
 		return err
 	}

 	client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin")))
-	msgModel := cache.NewMsgCacheModel(rdb)
-	msgDocModel := unrelation.NewMsgMongoDriver(mongo.GetDatabase())
-	msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel)
+	msgModel := cache.NewMsgCacheModel(rdb, config)
+	msgDocModel := unrelation.NewMsgMongoDriver(mongo.GetDatabase(config.Mongo.Database))
+	msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, config)
 	if err != nil {
 		return err
 	}
-	conversationRpcClient := rpcclient.NewConversationRpcClient(client)
-	groupRpcClient := rpcclient.NewGroupRpcClient(client)
-	msgTransfer, err := NewMsgTransfer(msgDatabase, &conversationRpcClient, &groupRpcClient)
+	conversationRpcClient := rpcclient.NewConversationRpcClient(client, config)
+	groupRpcClient := rpcclient.NewGroupRpcClient(client, config)
+	msgTransfer, err := NewMsgTransfer(config, msgDatabase, &conversationRpcClient, &groupRpcClient)
 	if err != nil {
 		return err
 	}
-	return msgTransfer.Start(prometheusPort)
+	return msgTransfer.Start(prometheusPort, config)
 }

-func NewMsgTransfer(msgDatabase controller.CommonMsgDatabase, conversationRpcClient *rpcclient.ConversationRpcClient, groupRpcClient *rpcclient.GroupRpcClient) (*MsgTransfer, error) {
-	historyCH, err := NewOnlineHistoryRedisConsumerHandler(msgDatabase, conversationRpcClient, groupRpcClient)
+func NewMsgTransfer(config *config.GlobalConfig, msgDatabase controller.CommonMsgDatabase, conversationRpcClient *rpcclient.ConversationRpcClient, groupRpcClient *rpcclient.GroupRpcClient) (*MsgTransfer, error) {
+	historyCH, err := NewOnlineHistoryRedisConsumerHandler(config, msgDatabase, conversationRpcClient, groupRpcClient)
 	if err != nil {
 		return nil, err
 	}
-	historyMongoCH, err := NewOnlineHistoryMongoConsumerHandler(msgDatabase)
+	historyMongoCH, err := NewOnlineHistoryMongoConsumerHandler(config, msgDatabase)
 	if err != nil {
 		return nil, err
 	}
@@ -105,11 +105,12 @@ func NewMsgTransfer(msgDatabase controller.CommonMsgDatabase, conversationRpcCli
 	return &MsgTransfer{
 		historyCH:      historyCH,
 		historyMongoCH: historyMongoCH,
+		config:         config,
 	}, nil
 }

-func (m *MsgTransfer) Start(prometheusPort int) error {
-	fmt.Println("Start msg transfer", "prometheusPort:", prometheusPort)
+func (m *MsgTransfer) Start(prometheusPort int, config *config.GlobalConfig) error {
+	fmt.Println("start msg transfer", "prometheusPort:", prometheusPort)
 	if prometheusPort <= 0 {
 		return errs.Wrap(errors.New("prometheusPort not correct"))
 	}
@@ -123,13 +124,13 @@ func (m *MsgTransfer) Start(prometheusPort int) error {
 	go m.historyCH.historyConsumerGroup.RegisterHandleAndConsumer(m.ctx, m.historyCH)
 	go m.historyMongoCH.historyConsumerGroup.RegisterHandleAndConsumer(m.ctx, m.historyMongoCH)

-	if config.Config.Prometheus.Enable {
+	if config.Prometheus.Enable {
 		go func() {
 			proreg := prometheus.NewRegistry()
 			proreg.MustRegister(
 				collectors.NewGoCollector(),
 			)
-			proreg.MustRegister(prommetrics.GetGrpcCusMetrics("Transfer")...)
+			proreg.MustRegister(prommetrics.GetGrpcCusMetrics("Transfer", config)...)
 			http.Handle("/metrics", promhttp.HandlerFor(proreg, promhttp.HandlerOpts{Registry: proreg}))
 			err := http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), nil)
 			if err != nil && err != http.ErrServerClosed {
@@ -81,6 +81,7 @@ type OnlineHistoryRedisConsumerHandler struct {
 }

 func NewOnlineHistoryRedisConsumerHandler(
+	config *config.GlobalConfig,
 	database controller.CommonMsgDatabase,
 	conversationRpcClient *rpcclient.ConversationRpcClient,
 	groupRpcClient *rpcclient.GroupRpcClient,
@@ -96,11 +97,29 @@ func NewOnlineHistoryRedisConsumerHandler(
 	och.conversationRpcClient = conversationRpcClient
 	och.groupRpcClient = groupRpcClient
 	var err error
+
+	var tlsConfig *kafka.TLSConfig
+	if config.Kafka.TLS != nil {
+		tlsConfig = &kafka.TLSConfig{
+			CACrt:              config.Kafka.TLS.CACrt,
+			ClientCrt:          config.Kafka.TLS.ClientCrt,
+			ClientKey:          config.Kafka.TLS.ClientKey,
+			ClientKeyPwd:       config.Kafka.TLS.ClientKeyPwd,
+			InsecureSkipVerify: false,
+		}
+	}
+
 	och.historyConsumerGroup, err = kafka.NewMConsumerGroup(&kafka.MConsumerGroupConfig{
 		KafkaVersion:   sarama.V2_0_0_0,
-		OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false,
-	}, []string{config.Config.Kafka.LatestMsgToRedis.Topic},
-		config.Config.Kafka.Addr, config.Config.Kafka.ConsumerGroupID.MsgToRedis)
+		OffsetsInitial: sarama.OffsetNewest,
+		IsReturnErr:    false,
+		UserName:       config.Kafka.Username,
+		Password:       config.Kafka.Password,
+	}, []string{config.Kafka.LatestMsgToRedis.Topic},
+		config.Kafka.Addr,
+		config.Kafka.ConsumerGroupID.MsgToRedis,
+		tlsConfig,
+	)
 	// statistics.NewStatistics(&och.singleMsgSuccessCount, config.Config.ModuleName.MsgTransferName, fmt.Sprintf("%d
 	// second singleMsgCount insert to mongo", constant.StatisticsTimeInterval), constant.StatisticsTimeInterval)
 	return &och, err
@@ -32,12 +32,28 @@ type OnlineHistoryMongoConsumerHandler struct {
 	msgDatabase          controller.CommonMsgDatabase
 }

-func NewOnlineHistoryMongoConsumerHandler(database controller.CommonMsgDatabase) (*OnlineHistoryMongoConsumerHandler, error) {
+func NewOnlineHistoryMongoConsumerHandler(config *config.GlobalConfig, database controller.CommonMsgDatabase) (*OnlineHistoryMongoConsumerHandler, error) {
+	var tlsConfig *kfk.TLSConfig
+	if config.Kafka.TLS != nil {
+		tlsConfig = &kfk.TLSConfig{
+			CACrt:              config.Kafka.TLS.CACrt,
+			ClientCrt:          config.Kafka.TLS.ClientCrt,
+			ClientKey:          config.Kafka.TLS.ClientKey,
+			ClientKeyPwd:       config.Kafka.TLS.ClientKeyPwd,
+			InsecureSkipVerify: false,
+		}
+	}
 	historyConsumerGroup, err := kfk.NewMConsumerGroup(&kfk.MConsumerGroupConfig{
 		KafkaVersion:   sarama.V2_0_0_0,
-		OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false,
-	}, []string{config.Config.Kafka.MsgToMongo.Topic},
-		config.Config.Kafka.Addr, config.Config.Kafka.ConsumerGroupID.MsgToMongo)
+		OffsetsInitial: sarama.OffsetNewest,
+		IsReturnErr:    false,
+		UserName:       config.Kafka.Username,
+		Password:       config.Kafka.Password,
+	}, []string{config.Kafka.MsgToMongo.Topic},
+		config.Kafka.Addr,
+		config.Kafka.ConsumerGroupID.MsgToMongo,
+		tlsConfig,
+	)
 	if err != nil {
 		return nil, err
 	}