fix: improve crontask delete outdated Data. (#2901)

* fix: improve crontask delete outdated Data.

* remove comments.

* update go pkg

* revert go pkg.

* update gRPC Implement.

* feat: support use config filter in deleteOutdatedData.

* update go pkg.

* update pkg.

* comment GOProxy.

internal/msggateway/hub_server.go

@@ -56,6 +56,7 @@ func (s *Server) Start(ctx context.Context, index int, conf *Config) error {
 }
 
 type Server struct {
+	msggateway.UnimplementedMsgGatewayServer
 	rpcPort        int
 	LongConnServer LongConnServer
 	config         *Config

internal/push/push.go

@@ -14,6 +14,7 @@ import (
 )
 
 type pushServer struct {
+	pbpush.UnimplementedPushMsgServiceServer
 	database      controller.PushDatabase
 	disCov        discovery.SvcDiscoveryRegistry
 	offlinePusher offlinepush.OfflinePusher

internal/rpc/auth/auth.go

@@ -40,6 +40,7 @@ import (
 )
 
 type authServer struct {
+	pbauth.UnimplementedAuthServer
 	authDatabase   controller.AuthDatabase
 	userRpcClient  *rpcclient.UserRpcClient
 	RegisterCenter discovery.SvcDiscoveryRegistry

internal/rpc/conversation/conversation.go

@@ -43,6 +43,7 @@ import (
 )
 
 type conversationServer struct {
+	pbconversation.UnimplementedConversationServer
 	msgRpcClient         *rpcclient.MessageRpcClient
 	user                 *rpcclient.UserRpcClient
 	groupRpcClient       *rpcclient.GroupRpcClient

internal/rpc/group/group.go

@@ -57,6 +57,7 @@ import (
 )
 
 type groupServer struct {
+	pbgroup.UnimplementedGroupServer
 	db                    controller.GroupDatabase
 	user                  rpcclient.UserRpcClient
 	notification          *GroupNotificationSender

internal/rpc/msg/server.go

@@ -55,6 +55,7 @@ type (
 		msgNotificationSender  *MsgNotificationSender           // RPC client for sending msg notifications.
 		config                 *Config                          // Global configuration settings.
 		webhookClient          *webhook.Client
+		msg.UnimplementedMsgServer
 	}
 
 	Config struct {

internal/rpc/relation/friend.go

@@ -43,6 +43,7 @@ import (
 )
 
 type friendServer struct {
+	relation.UnimplementedFriendServer
 	db                    controller.FriendDatabase
 	blackDatabase         controller.BlackDatabase
 	userRpcClient         *rpcclient.UserRpcClient

internal/rpc/third/s3.go

@@ -290,48 +290,85 @@ func (t *thirdServer) apiAddress(prefix, name string) string {
 func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) {
 	var conf config.Third
 	expireTime := time.UnixMilli(req.ExpireTime)
-	var deltotal int
+
 	findPagination := &sdkws.RequestPagination{
 		PageNumber: 1,
-		ShowNumber: 1000,
+		ShowNumber: 500,
 	}
-	for {
-		total, models, err := t.s3dataBase.FindByExpires(ctx, expireTime, findPagination)
+
+	// Find all expired data in S3 database
+	total, models, err := t.s3dataBase.FindNeedDeleteObjectByDB(ctx, expireTime, req.ObjectGroup, findPagination)
+	if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments {
+		return nil, errs.Wrap(err)
+	}
+
+	if total == 0 {
+		log.ZDebug(ctx, "Not have OutdatedData", "delete Total", total)
+		return &third.DeleteOutdatedDataResp{Count: int32(total)}, nil
+	}
+
+	needDelObjectKeys := make([]string, len(models))
+	for _, model := range models {
+		needDelObjectKeys = append(needDelObjectKeys, model.Key)
+	}
+
+	// Remove duplicate keys, have the same key use in different models
+	needDelObjectKeys = datautil.Distinct(needDelObjectKeys)
+
+	for _, key := range needDelObjectKeys {
+		// Find all models by key
+		keyModels, err := t.s3dataBase.FindModelsByKey(ctx, key)
 		if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments {
 			return nil, errs.Wrap(err)
 		}
-		needDelObjectKeys := make([]string, 0)
-		for _, model := range models {
-			needDelObjectKeys = append(needDelObjectKeys, model.Key)
-		}
 
-		needDelObjectKeys = datautil.Distinct(needDelObjectKeys)
-		for _, key := range needDelObjectKeys {
-			count, err := t.s3dataBase.FindNotDelByS3(ctx, key, expireTime)
-			if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments {
-				return nil, errs.Wrap(err)
-			}
-			if int(count) < 1 && t.minio != nil {
-				thumbnailKey, _ := t.getMinioImageThumbnailKey(ctx, key)
-
-				t.s3dataBase.DeleteObject(ctx, thumbnailKey)
-				t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, needDelObjectKeys...)
-				t.s3dataBase.DeleteObject(ctx, key)
+		// check keyModels, if all keyModels.
+		needDelKey := true // Default can delete
+		for _, keymodel := range keyModels {
+			// If group is empty or CreateTime is after expireTime, can't delete this key
+			if keymodel.Group == "" || keymodel.CreateTime.After(expireTime) {
+				needDelKey = false
+				break
 			}
 		}
-		for _, model := range models {
-			err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name)
+
+		// If this object is not referenced by not expire data, delete it
+		if needDelKey && t.minio != nil {
+			// If have a thumbnail, delete it
+			thumbnailKey, _ := t.getMinioImageThumbnailKey(ctx, key)
+			if thumbnailKey != "" {
+				err := t.s3dataBase.DeleteObject(ctx, thumbnailKey)
+				if err != nil {
+					log.ZWarn(ctx, "Delete thumbnail object is error:", errs.Wrap(err), "thumbnailKey", thumbnailKey)
+				}
+			}
+
+			// Delete object
+			err = t.s3dataBase.DeleteObject(ctx, key)
 			if err != nil {
-				return nil, errs.Wrap(err)
+				log.ZWarn(ctx, "Delete object is error", errs.Wrap(err), "object key", key)
+			}
+
+			// Delete cache key
+			err = t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, key)
+			if err != nil {
+				log.ZWarn(ctx, "Delete cache key is error:", errs.Wrap(err), "cache S3 key:", key)
 			}
 		}
-		if total < int64(findPagination.ShowNumber) {
-			break
-		}
-		deltotal += int(total)
 	}
-	log.ZDebug(ctx, "DeleteOutdatedData", "delete Total", deltotal)
-	return &third.DeleteOutdatedDataResp{}, nil
+
+	// handle delete data in S3 database
+	for _, model := range models {
+		// Delete all expired data row in S3 database
+		err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name)
+		if err != nil {
+			return nil, errs.Wrap(err)
+		}
+	}
+
+	log.ZDebug(ctx, "DeleteOutdatedData", "delete Total", total)
+
+	return &third.DeleteOutdatedDataResp{Count: int32(total)}, nil
 }
 
 type FormDataMate struct {

internal/rpc/third/third.go

@@ -38,6 +38,7 @@ import (
 )
 
 type thirdServer struct {
+	third.UnimplementedThirdServer
 	thirdDatabase controller.ThirdDatabase
 	s3dataBase    controller.S3Database
 	userRpcClient rpcclient.UserRpcClient

internal/rpc/user/user.go

@@ -52,6 +52,7 @@ import (
 )
 
 type userServer struct {
+	pbuser.UnimplementedUserServer
 	online                   cache.OnlineCache
 	db                       controller.UserDatabase
 	friendNotificationSender *relation.FriendNotificationSender

internal/tools/cron_task.go

@@ -24,6 +24,7 @@ import (
 	kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister"
 	pbconversation "github.com/openimsdk/protocol/conversation"
 	"github.com/openimsdk/protocol/msg"
+	"github.com/openimsdk/protocol/third"
 
 	"github.com/openimsdk/tools/mcontext"
 	"github.com/openimsdk/tools/mw"
@@ -58,10 +59,10 @@ func Start(ctx context.Context, config *CronTaskConfig) error {
 		return err
 	}
 
-	// thirdConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third)
-	// if err != nil {
-	// 	return err
-	// }
+	thirdConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third)
+	if err != nil {
+		return err
+	}
 
 	conversationConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Conversation)
 	if err != nil {
@@ -70,7 +71,7 @@ func Start(ctx context.Context, config *CronTaskConfig) error {
 
 	msgClient := msg.NewMsgClient(msgConn)
 	conversationClient := pbconversation.NewConversationClient(conversationConn)
-	// thirdClient := third.NewThirdClient(thirdConn)
+	thirdClient := third.NewThirdClient(thirdConn)
 
 	crontab := cron.New()
 
@@ -114,21 +115,37 @@ func Start(ctx context.Context, config *CronTaskConfig) error {
 		return errs.Wrap(err)
 	}
 
-	// // scheduled delete outdated file Objects and their datas in specific time.
-	// deleteObjectFunc := func() {
-	// 	now := time.Now()
-	// 	deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime))
-	// 	ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli()))
-	// 	log.ZDebug(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli())
-	// 	if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil {
-	// 		log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now))
-	// 		return
-	// 	}
-	// 	log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now))
-	// }
-	// if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil {
-	// 	return errs.Wrap(err)
-	// }
+	// scheduled delete outdated file Objects and their datas in specific time.
+	deleteObjectFunc := func() {
+		now := time.Now()
+		executeNum := 5
+		// number of pagination. if need modify, need update value in third.DeleteOutdatedData
+		pageShowNumber := 500
+		deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime))
+		ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli()))
+		log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli())
+
+		if len(config.CronTask.DeleteObjectType) == 0 {
+			log.ZDebug(ctx, "cron deleteoutDatedData not type need delete", "deletetime", deleteTime, "DeleteObjectType", config.CronTask.DeleteObjectType, "cont", time.Since(now))
+			return
+		}
+
+		for i := 0; i < executeNum; i++ {
+			resp, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: config.CronTask.DeleteObjectType})
+			if err != nil {
+				log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now))
+				return
+			}
+			if resp.Count == 0 || resp.Count < int32(pageShowNumber) {
+				break
+			}
+		}
+
+		log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now))
+	}
+	if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil {
+		return errs.Wrap(err)
+	}
 
 	log.ZDebug(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime)
 	crontab.Start()