rewrite publishing-bot verify script in go

- setup scripts to trigger publishing-bot verify
- add golang verify script and remove python verification code
- pin k8s.io/publishing-bot to v0.5.0

Signed-off-by: Akhil Mohan <akhilerm@gmail.com>

hack/tools/go.mod

@@ -10,8 +10,10 @@ require (
 	github.com/jcchavezs/porto v0.6.0
 	github.com/vektra/mockery/v2 v2.40.3
 	go.uber.org/automaxprocs v1.5.2
+	golang.org/x/mod v0.20.0
 	gotest.tools/gotestsum v1.12.0
 	honnef.co/go/tools v0.5.1
+	k8s.io/publishing-bot v0.5.0
 	sigs.k8s.io/logtools v0.8.1
 )
 
@@ -74,6 +76,7 @@ require (
 	github.com/go-xmlfmt/xmlfmt v1.1.2 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/golang/glog v1.2.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
 	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
@@ -190,7 +193,6 @@ require (
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
 	golang.org/x/exp/typeparams v0.0.0-20231219180239-dc181d75b848 // indirect
-	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/term v0.18.0 // indirect

hack/tools/go.sum

@@ -207,6 +207,8 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY=
+github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -678,8 +680,8 @@ golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1024,6 +1026,8 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 honnef.co/go/tools v0.5.1 h1:4bH5o3b5ZULQ4UrBmP+63W9r7qIkqJClEA9ko5YKx+I=
 honnef.co/go/tools v0.5.1/go.mod h1:e9irvo83WDG9/irijV44wr3tbhcFeRnfpVlRqVwpzMs=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/publishing-bot v0.5.0 h1:Hfnhltr+khEcqvoK4GBYrtaA8dHJ50Xjyi+0KGUfU3I=
+k8s.io/publishing-bot v0.5.0/go.mod h1:S5+zQQhsVUEqdcaohbYf8O+2BeeWRtuYzp4tQLr5An8=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 mvdan.cc/gofumpt v0.6.0 h1:G3QvahNDmpD+Aek/bNOLrFR2XC6ZAdo62dZu65gmwGo=
 mvdan.cc/gofumpt v0.6.0/go.mod h1:4L0wf+kgIPZtcCWXynNS2e6bhmj73umwnuXSZarixzA=

hack/tools/publishing-verifier/publishing-verifier.go

@@ -0,0 +1,240 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/mod/modfile"
+	"k8s.io/publishing-bot/cmd/publishing-bot/config"
+)
+
+var (
+	rulesFile           string
+	componentsDirectory string
+)
+
+// getGoModDependencies gets all the staging dependencies for all the modules
+// in the given directory
+func getGoModDependencies(dir string) (map[string][]string, error) {
+	allDependencies := make(map[string][]string)
+	components, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+	for _, component := range components {
+		componentName := component.Name()
+		if !component.IsDir() {
+			// currently there is no hard check that the staging directory should not contain
+			// other files
+			continue
+		}
+		gomodFilePath := filepath.Join(dir, componentName, "go.mod")
+		gomodFileContent, err := os.ReadFile(gomodFilePath)
+		if err != nil {
+			return nil, err
+		}
+
+		fmt.Printf("%s dependencies", componentName)
+
+		allDependencies[componentName] = make([]string, 0)
+
+		gomodFile, err := modfile.Parse(gomodFilePath, gomodFileContent, nil)
+		if err != nil {
+			return nil, err
+		}
+		// get all the other dependencies from within staging, i.e all the modules in replace
+		// section
+		for _, module := range gomodFile.Replace {
+			dep := strings.TrimPrefix(module.Old.Path, "k8s.io/")
+			if dep == componentName {
+				continue
+			}
+			allDependencies[componentName] = append(allDependencies[componentName], dep)
+		}
+	}
+	return allDependencies, nil
+}
+
+// diffSlice returns the difference of s1-s2
+func diffSlice(s1, s2 []string) []string {
+	var diff []string
+	set := make(map[string]struct{}, len(s2))
+	for _, s := range s2 {
+		set[s] = struct{}{}
+	}
+	for _, s := range s1 {
+		if _, ok := set[s]; !ok {
+			diff = append(diff, s)
+		}
+	}
+	return diff
+}
+
+// getKeys returns a slice with only the keys of the given map
+func getKeys[K comparable, V any](m map[K]V) []K {
+	var keys []K
+	for k := range m {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+// checkValidSourceDirectory checks if proper source directory fields are used in rules
+func checkValidSourceDirectory(rule config.RepositoryRule) error {
+	for _, branch := range rule.Branches {
+		if branch.Source.Dir != "" {
+			return fmt.Errorf("use of deprecated `dir` field in rules for `%s`", rule.DestinationRepository)
+		}
+		if len(branch.Source.Dirs) > 1 {
+			return fmt.Errorf("cannot have more than one directory (%s) per source branch `%s` of `%s`",
+				branch.Source.Dirs,
+				branch.Source.Branch,
+				rule.DestinationRepository,
+			)
+		}
+		if !strings.HasSuffix(branch.Source.Dirs[0], rule.DestinationRepository) {
+			return fmt.Errorf("copy/paste error `%s` refers to `%s`", rule.DestinationRepository, branch.Source.Dirs[0])
+		}
+	}
+	return nil
+}
+
+// checkMasterBranch checks if the master branch of destination repository refers to the master
+// of the source
+func checkMasterBranch(rule config.RepositoryRule) error {
+	branch := rule.Branches[0]
+	if branch.Name != "master" {
+		return fmt.Errorf("cannot find master branch for destination `%s`", rule.DestinationRepository)
+	}
+
+	if branch.Source.Branch != "master" {
+		return fmt.Errorf("cannot find master source branch for destination `%s`", rule.DestinationRepository)
+	}
+	return nil
+}
+
+func checkDependencies(rule config.RepositoryRule, gomodDependencies map[string][]string) error {
+	var processedDeps []string
+	branch := rule.Branches[0]
+	for _, dep := range gomodDependencies[rule.DestinationRepository] {
+		found := false
+		if len(branch.Dependencies) > 0 {
+			for _, dep2 := range branch.Dependencies {
+				processedDeps = append(processedDeps, dep2.Repository)
+				if dep2.Branch != "master" {
+					return fmt.Errorf("looking for master branch of %s and found : %s for destination", dep2.Repository, rule.DestinationRepository)
+				}
+				if dep2.Repository == dep {
+					found = true
+				}
+			}
+		} else {
+			return fmt.Errorf("Please add %s as dependencies under destination %s", gomodDependencies[rule.DestinationRepository], rule.DestinationRepository)
+		}
+		if !found {
+			return fmt.Errorf("Please add %s as a dependency under destination %s", dep, rule.DestinationRepository)
+		} else {
+			fmt.Printf("dependency %s found\n", dep)
+		}
+	}
+	// check if all deps are processed.
+	extraDeps := diffSlice(processedDeps, gomodDependencies[rule.DestinationRepository])
+	if len(extraDeps) > 0 {
+		return fmt.Errorf("extra dependencies in rules for %s: %s", rule.DestinationRepository, strings.Join(extraDeps, ","))
+	}
+	return nil
+}
+
+func verifyPublishingBotRules() error {
+	rules, err := config.LoadRules(rulesFile)
+	if err != nil {
+		return fmt.Errorf("error loading rules: %v", err)
+	}
+
+	gomodDependencies, err := getGoModDependencies(componentsDirectory)
+
+	var processedRepos []string
+	for _, rule := range rules.Rules {
+		branch := rule.Branches[0]
+
+		// if this no longer exists in master
+		if _, ok := gomodDependencies[rule.DestinationRepository]; !ok {
+			// make sure we dont include a rule to publish it from master
+			for _, branch := range rule.Branches {
+				if branch.Name == "master" {
+					err := fmt.Errorf("cannot find master branch for destination `%s`", rule.DestinationRepository)
+					panic(err)
+				}
+			}
+			// and skip the validation of publishing rules for it
+			continue
+		}
+
+		if err := checkValidSourceDirectory(rule); err != nil {
+			return fmt.Errorf("error validating source directory: %v", err)
+		}
+
+		if err := checkMasterBranch(rule); err != nil {
+			return fmt.Errorf("error validating master branch: %v", err)
+		}
+
+		// we specify the go version for all master branches through `default-go-version`
+		// so ensure we don't specify explicit go version for master branch in rules
+		if branch.GoVersion != "" {
+			err := fmt.Errorf("go version must not be specified for master branch for destination `%s`", rule.DestinationRepository)
+			panic(err)
+		}
+
+		fmt.Printf("processing : %s", rule.DestinationRepository)
+		if _, ok := gomodDependencies[rule.DestinationRepository]; !ok {
+			err := fmt.Errorf("missing go.mod for `%s`", rule.DestinationRepository)
+			panic(err)
+		}
+		processedRepos = append(processedRepos, rule.DestinationRepository)
+
+		if err := checkDependencies(rule, gomodDependencies); err != nil {
+			return fmt.Errorf("error validating dependencies: %v", err)
+		}
+	}
+
+	// check if all repos are processed.
+	items := diffSlice(getKeys(gomodDependencies), processedRepos)
+	if len(items) > 0 {
+		err := fmt.Errorf("missing rules for %s", strings.Join(items, ","))
+		panic(err)
+	}
+	return nil
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		panic("invalid number of arguments")
+	}
+
+	kubeRoot := os.Args[1]
+	stagingDirectory := kubeRoot + "/staging/"
+	rulesFile = stagingDirectory + "publishing/rules.yaml"
+	componentsDirectory = stagingDirectory + "src/k8s.io/"
+
+	if err := verifyPublishingBotRules(); err != nil {
+		panic(err)
+	}
+}

hack/tools/tools.go

@@ -36,4 +36,8 @@ import (
 
 	// tools like cpu
 	_ "go.uber.org/automaxprocs"
+
+	// for publishing bot
+	_ "golang.org/x/mod/modfile"
+	_ "k8s.io/publishing-bot/cmd/publishing-bot/config"
 )