feat: Enhancements to OpenIM Engineering Practices with Standardizer and Tool Versioning (#2159)

* feat: add standardizer optimize makefile

* feat: add standardizer optimize makefile

* feat: add openim test docs

* feat: add openim kafka docs

* feat: add openim kafka docs

* feat: add openim kafka docs

scripts/lib/golang.sh

@@ -314,9 +314,6 @@ openim::golang::setup_platforms
 readonly OPENIM_CLIENT_TARGETS=(
     changelog
     component
-    conversion-msg
-    conversion-mysql
-    formitychecker
     imctl
     infra
     ncpu

scripts/lib/release.sh

@@ -243,7 +243,7 @@ function openim::release::package_client_tarballs() {
 
     local client_bins=("${OPENIM_CLIENT_BINARIES[@]}")
 
-    # client_bins: changelog component conversion-msg conversion-mysql formitychecker imctl infra ncpu openim-web up35 versionchecker yamlfmt
+    # client_bins: changelog component imctl infra ncpu openim-web up35 versionchecker yamlfmt
     # Copy client binclient_bins:aries
     openim::log::info " Copy client binaries: ${client_bins[@]/#/${LOCAL_OUTPUT_BINTOOLSPATH}/${platform}/}"
     openim::log::info " Copy client binaries to: ${release_stage}/client/bin"

scripts/make-rules/common-versions.mk

@@ -0,0 +1,58 @@
+# Copyright © 2023 OpenIMSDK.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# OpenIM Makefile Versions used
+#
+# Define the latest version for each tool to ensure consistent versioning across installations
+GOLANGCI_LINT_VERSION ?= latest
+GOIMPORTS_VERSION ?= latest
+ADDLICENSE_VERSION ?= latest
+DEEPCOPY_GEN_VERSION ?= latest
+CONVERSION_GEN_VERSION ?= latest
+GINKGO_VERSION ?= v1.16.2
+GO_GITLINT_VERSION ?= latest
+GO_JUNIT_REPORT_VERSION ?= latest
+GOTESTS_VERSION ?= latest
+SWAGGER_VERSION ?= latest
+KUBE_SCORE_VERSION ?= latest
+KUBECONFORM_VERSION ?= latest
+GSEMVER_VERSION ?= latest
+GIT_CHGLOG_VERSION ?= latest
+KO_VERSION ?= latest
+GITHUB_RELEASE_VERSION ?= latest
+COSCLI_VERSION ?= v0.19.0-beta
+MINIO_VERSION ?= latest
+DELVE_VERSION ?= latest
+AIR_VERSION ?= latest
+GOLINES_VERSION ?= latest
+GO_MOD_OUTDATED_VERSION ?= latest
+CFSSL_VERSION ?= latest
+DEPTH_VERSION ?= latest
+GO_CALLVIS_VERSION ?= latest
+MISSPELL_VERSION ?= latest
+GOTHANKS_VERSION ?= latest
+RICHGO_VERSION ?= latest
+RTS_VERSION ?= latest
+TYPECHECK_VERSION ?= latest
+COMMENT_LANG_DETECTOR_VERSION ?= latest
+STANDARDIZER_VERSION ?= latest
+GO_TESTS_VERSION ?= v1.6.0
+GO_APIDIFF_VERSION ?= v0.8.2
+KAFKACTL_VERSION ?= latest
+GOTESTSUM_VERSION ?= latest
+
+WIRE_VERSION ?= latest
+# WIRE_VERSION ?= $(call get_go_version,github.com/google/wire)
+MOCKGEN_VERSION ?= $(call get_go_version,github.com/golang/mock)
+PROTOC_GEN_GO_VERSION ?= $(call get_go_version,github.com/golang/protobuf/protoc-gen-go)

scripts/make-rules/common.mk

@@ -78,6 +78,13 @@ VERSION := $(shell git describe --tags --always --match='v*')
 # v2.3.3: git tag
 endif
 
+# Helper function to get dependency version from go.mod
+get_gomod_version = $(shell go list -m $1 | awk '{print $$2}')
+define go_install
+$(info ===========> Installing $(1)@$(2))
+$(GO) install $(1)@$(2)
+endef
+
 # Check if the tree is dirty. default to dirty(maybe u should commit?)
 GIT_TREE_STATE:="dirty"
 ifeq (, $(shell git status --porcelain 2>/dev/null))

scripts/make-rules/tools.mk

@@ -64,81 +64,101 @@ tools.verify.%:
 ## install.golangci-lint: Install golangci-lint
 .PHONY: install.golangci-lint
 install.golangci-lint:
-	@$(GO) install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
+	@$(GO) install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
 
 ## install.goimports: Install goimports, used to format go source files
 .PHONY: install.goimports
 install.goimports:
-	@$(GO) install golang.org/x/tools/cmd/goimports@latest
+	@$(GO) install golang.org/x/tools/cmd/goimports@$(GOIMPORTS_VERSION)
 
 ## install.addlicense: Install addlicense, used to add license header to source files
 .PHONY: install.addlicense
 install.addlicense:
-	@$(GO) install github.com/google/addlicense@latest
+	@$(GO) install github.com/google/addlicense@$(ADDLICENSE_VERSION)
 
 ## install.deepcopy-gen: Install deepcopy-gen, used to generate deep copy functions
 .PHONY: install.deepcopy-gen
 install.deepcopy-gen:
-	@$(GO) install k8s.io/code-generator/cmd/deepcopy-gen@latest
+	@$(GO) install k8s.io/code-generator/cmd/deepcopy-gen@$(DEEPCOPY_GEN_VERSION)
 
 ## install.conversion-gen: Install conversion-gen, used to generate conversion functions
 .PHONY: install.conversion-gen
 install.conversion-gen:
-	@$(GO) install k8s.io/code-generator/cmd/conversion-gen@latest
+	@$(GO) install k8s.io/code-generator/cmd/conversion-gen@$(CONVERSION_GEN_VERSION)
 
 ## install.ginkgo: Install ginkgo to run a single test or set of tests
 .PHONY: install.ginkgo
 install.ginkgo:
-	@$(GO) install github.com/onsi/ginkgo/ginkgo@v1.16.2
+	@$(GO) install github.com/onsi/ginkgo/ginkgo@$(GINKGO_VERSION)
 
-## Install go-gitlint: Install go-gitlint, used to check git commit message
+## install.go-gitlint: Install go-gitlint, used to check git commit message
 .PHONY: install.go-gitlint
 install.go-gitlint:
-	@$(GO) install github.com/marmotedu/go-gitlint/cmd/go-gitlint@latest
+	@$(GO) install github.com/marmotedu/go-gitlint/cmd/go-gitlint@$(GO_GITLINT_VERSION)
 
 ## install.go-junit-report: Install go-junit-report, used to convert go test output to junit xml
 .PHONY: install.go-junit-report
 install.go-junit-report:
-	@$(GO) install github.com/jstemmer/go-junit-report@latest
+	@$(GO) install github.com/jstemmer/go-junit-report@$(GO_JUNIT_REPORT_VERSION)
 
 ## install.gotests: Install gotests, used to generate go tests
+.PHONY: install.gotests
+install.gotests:
+	@$(GO) install github.com/cweill/gotests/gotests@$(GO_TESTS_VERSION)
+
+## install.kafkactl: Install kafkactl command line tool.
+.PHONY: install.kafkactl
+install.kafkactl:
+	@$(GO) install github.com/deviceinsight/kafkactl@$(KAFKACTL_VERSION)
+
+## install.go-apidiff: Install go-apidiff, used to check api changes
+.PHONY: install.go-apidiff
+install.go-apidiff:
+	@$(GO) install github.com/joelanford/go-apidiff@$(GO_APIDIFF_VERSION)
+
+## install.swagger: Install swagger, used to generate swagger documentation
 .PHONY: install.swagger
 install.swagger:
-	@$(GO) install github.com/go-swagger/go-swagger/cmd/swagger@latest
+	@$(GO) install github.com/go-swagger/go-swagger/cmd/swagger@$(SWAGGER_VERSION)
 
 # ==============================================================================
 # Tools that might be used include go gvm
 #
 
+## install.gotestsum: Install gotestsum, used to run go tests
+.PHONY: install.gotestsum
+install.gotestsum:
+	@$(GO) install gotest.tools/gotestsum@$(GOTESTSUM_VERSION)
+
 ## install.kube-score: Install kube-score, used to check kubernetes yaml files
 .PHONY: install.kube-score
 install.kube-score:
-	@$(GO) install github.com/zegl/kube-score/cmd/kube-score@latest
+	@$(GO) install github.com/zegl/kube-score/cmd/kube-score@$(KUBE_SCORE_VERSION)
 
 ## install.kubeconform: Install kubeconform, used to check kubernetes yaml files
 .PHONY: install.kubeconform
 install.kubeconform:
-	@$(GO) install github.com/yannh/kubeconform/cmd/kubeconform@latest
+	@$(GO) install github.com/yannh/kubeconform/cmd/kubeconform@$(KUBECONFORM_VERSION)
 
 ## install.gsemver: Install gsemver, used to generate semver
 .PHONY: install.gsemver
 install.gsemver:
-	@$(GO) install github.com/arnaud-deprez/gsemver@latest
+	@$(GO) install github.com/arnaud-deprez/gsemver@$(GSEMVER_VERSION)
 
 ## install.git-chglog: Install git-chglog, used to generate changelog
 .PHONY: install.git-chglog
 install.git-chglog:
-	@$(GO) install github.com/git-chglog/git-chglog/cmd/git-chglog@latest
+	@$(GO) install github.com/git-chglog/git-chglog/cmd/git-chglog@$(GIT_CHGLOG_VERSION)
 
 ## install.ko: Install ko, used to build go program into container images
 .PHONY: install.ko
 install.ko:
-	@$(GO) install github.com/google/ko@latest
+	@$(GO) install github.com/google/ko@$(KO_VERSION)
 
 ## install.github-release: Install github-release, used to create github release
 .PHONY: install.github-release
 install.github-release:
-	@$(GO) install github.com/github-release/github-release@latest
+	@$(GO) install github.com/github-release/github-release@$(GITHUB_RELEASE_VERSION)
 
 ## install.coscli: Install coscli, used to upload files to cos
 # example: ./coscli  cp/sync -r  /home/off-line/docker-off-line/ cos://openim-1306374445/openim/image/amd/off-line/off-line/ -e cos.ap-guangzhou.myqcloud.com
@@ -146,7 +166,7 @@ install.github-release:
 # amd64
 .PHONY: install.coscli
 install.coscli:
-	@wget -q https://github.com/tencentyun/coscli/releases/download/v0.19.0-beta/coscli-linux -O ${TOOLS_DIR}/coscli
+	@wget -q https://github.com/tencentyun/coscli/releases/download/$(COSCLI_VERSION)/coscli-linux -O ${TOOLS_DIR}/coscli
 	@chmod +x ${TOOLS_DIR}/coscli
 
 ## install.coscmd: Install coscmd, used to upload files to cos
@@ -157,50 +177,50 @@ install.coscmd:
 ## install.minio: Install minio, used to upload files to minio
 .PHONY: install.minio
 install.minio:
-	@$(GO) install github.com/minio/minio@latest
+	@$(GO) install github.com/minio/minio@$(MINIO_VERSION)
 
 ## install.delve: Install delve, used to debug go program
 .PHONY: install.delve
 install.delve:
-	@$(GO) install github.com/go-delve/delve/cmd/dlv@latest
+	@$(GO) install github.com/go-delve/delve/cmd/dlv@$(DELVE_VERSION)
 
 ## install.air: Install air, used to hot reload go program
 .PHONY: install.air
 install.air:
-	@$(GO) install github.com/cosmtrek/air@latest
+	@$(GO) install github.com/cosmtrek/air@$(AIR_VERSION)
 
 ## install.gvm: Install gvm, gvm is a Go version manager, built on top of the official go tool.
-# github: https://github.com/moovweb/gvm
 .PHONY: install.gvm
 install.gvm:
-	@echo "===========> Installing gvm,The default installation path is ~/.gvm/scripts/gvm"
+	@echo "===========> Installing gvm, The default installation path is ~/.gvm/scripts/gvm"
 	@bash < <(curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/master/binscripts/gvm-installer)
-	@$(shell source /root/.gvm/scripts/gvm)
+	@source /root/.gvm/scripts/gvm
 
 ## install.golines: Install golines, used to format long lines
 .PHONY: install.golines
 install.golines:
-	@$(GO) install github.com/segmentio/golines@latest
+	@$(GO) install github.com/segmentio/golines@$(GOLINES_VERSION)
 
 ## install.go-mod-outdated: Install go-mod-outdated, used to check outdated dependencies
 .PHONY: install.go-mod-outdated
 install.go-mod-outdated:
-	@$(GO) install github.com/psampaz/go-mod-outdated@latest
+	@$(GO) install github.com/psampaz/go-mod-outdated@$(GO_MOD_OUTDATED_VERSION)
 
 ## install.mockgen: Install mockgen, used to generate mock functions
 .PHONY: install.mockgen
 install.mockgen:
-	@$(GO) install github.com/golang/mock/mockgen@latest
+	@$(GO) install github.com/golang/mock/mockgen@$(MOCKGEN_VERSION)
+
+## install.wire: Install wire, used to generate wire files
+.PHONY: install.wire
+install.wire:
+	@$(GO) install github.com/google/wire/cmd/wire@$(WIRE_VERSION)
 
-## install.gotests: Install gotests, used to generate test functions
-.PHONY: install.gotests
-install.gotests:
-	@$(GO) install github.com/cweill/gotests/gotests@latest
 
 ## install.protoc-gen-go: Install protoc-gen-go, used to generate go source files from protobuf files
 .PHONY: install.protoc-gen-go
 install.protoc-gen-go:
-	@$(GO) install github.com/golang/protobuf/protoc-gen-go@latest
+	@$(GO) install github.com/golang/protobuf/protoc-gen-go@$(PROTOC_GEN_GO_VERSION)
 
 ## install.cfssl: Install cfssl, used to generate certificates
 .PHONY: install.cfssl
@@ -210,43 +230,49 @@ install.cfssl:
 ## install.depth: Install depth, used to check dependency tree
 .PHONY: install.depth
 install.depth:
-	@$(GO) install github.com/KyleBanks/depth/cmd/depth@latest
+	@$(GO) install github.com/KyleBanks/depth/cmd/depth@$(DEPTH_VERSION)
 
 ## install.go-callvis: Install go-callvis, used to visualize call graph
 .PHONY: install.go-callvis
 install.go-callvis:
-	@$(GO) install github.com/ofabry/go-callvis@latest
+	@$(GO) install github.com/ofabry/go-callvis@$(GO_CALLVIS_VERSION)
 
-## install.misspell
+## install.misspell: Install misspell
 .PHONY: install.misspell
 install.misspell:
-	@$(GO) install github.com/client9/misspell/cmd/misspell@latest
+	@$(GO) install github.com/client9/misspell/cmd/misspell@$(MISSPELL_VERSION)
 
 ## install.gothanks: Install gothanks, used to thank go dependencies
 .PHONY: install.gothanks
 install.gothanks:
-	@$(GO) install github.com/psampaz/gothanks@latest
+	@$(GO) install github.com/psampaz/gothanks@$(GOTHANKS_VERSION)
 
 ## install.richgo: Install richgo
 .PHONY: install.richgo
 install.richgo:
-	@$(GO) install github.com/kyoh86/richgo@latest
+	@$(GO) install github.com/kyoh86/richgo@$(RICHGO_VERSION)
 
 ## install.rts: Install rts
 .PHONY: install.rts
 install.rts:
-	@$(GO) install github.com/galeone/rts/cmd/rts@latest
+	@$(GO) install github.com/galeone/rts/cmd/rts@$(RTS_VERSION)
 
 # ================= kubecub openim tools =========================================
-## install.typecheck: install kubecub typecheck check for go code
+# https://github.com/kubecub
+## install.typecheck: Install kubecub typecheck, checks for go code
 .PHONY: install.typecheck
 install.typecheck:
-	@$(GO) install github.com/kubecub/typecheck@latest
+	@$(GO) install github.com/kubecub/typecheck@$(TYPECHECK_VERSION)
 
-## install.comment-lang-detector: install kubecub comment-lang-detector check for go code comment language
+## install.comment-lang-detector: Install kubecub comment-lang-detector, checks for go code comment language
 .PHONY: install.comment-lang-detector
 install.comment-lang-detector:
-	@$(GO) install github.com/kubecub/comment-lang-detector/cmd/cld@latest
+	@$(GO) install github.com/kubecub/comment-lang-detector/cmd/cld@$(COMMENT_LANG_DETECTOR_VERSION)
+
+## install.standardizer: Install kubecub standardizer, checks for go code standardization
+.PHONY: install.standardizer
+install.standardizer:
+	@$(GO) install github.com/kubecub/standardizer@$(STANDARDIZER_VERSION)
 
 ## tools.help: Display help information about the tools package
 .PHONY: tools.help

scripts/verify-standardizer.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Copyright © 2023 OpenIM. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script does a fast type check of script srnetes code for all platforms.
+# Usage: `scripts/verify-standardizer.sh`.
+
+OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${OPENIM_ROOT}/scripts/lib/init.sh"
+
+openim::golang::verify_go_version
+
+cd "${OPENIM_ROOT}"
+ret=0
+scripts/run-in-gopath.sh \
+make tools.verify.standardizer
+${OPENIM_ROOT}/_output/tools/standardizer || ret=$?
+if [[ $ret -ne 0 ]]; then
+  openim::log::error "Failed to check the directory name or file name. Your name may not meet the specification. Please check the configuration file and the directory or file name." >&2
+  openim::log::error "Please see https://github.com/kubecub/standardizer for more information." >&2
+  exit 1
+fi