rewrite publishing-bot verify script in go

- setup scripts to trigger publishing-bot verify
- add golang verify script and remove python verification code
- pin k8s.io/publishing-bot to v0.5.0

Signed-off-by: Akhil Mohan <akhilerm@gmail.com>

hack/tools/go.mod

@@ -10,8 +10,10 @@ require (
 	github.com/jcchavezs/porto v0.6.0
 	github.com/vektra/mockery/v2 v2.40.3
 	go.uber.org/automaxprocs v1.5.2
+	golang.org/x/mod v0.20.0
 	gotest.tools/gotestsum v1.12.0
 	honnef.co/go/tools v0.5.1
+	k8s.io/publishing-bot v0.5.0
 	sigs.k8s.io/logtools v0.8.1
 )
 
@@ -74,6 +76,7 @@ require (
 	github.com/go-xmlfmt/xmlfmt v1.1.2 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/golang/glog v1.2.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
 	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
@@ -190,7 +193,6 @@ require (
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
 	golang.org/x/exp/typeparams v0.0.0-20231219180239-dc181d75b848 // indirect
-	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/term v0.18.0 // indirect

hack/tools/go.sum

@@ -207,6 +207,8 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY=
+github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -678,8 +680,8 @@ golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1024,6 +1026,8 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 honnef.co/go/tools v0.5.1 h1:4bH5o3b5ZULQ4UrBmP+63W9r7qIkqJClEA9ko5YKx+I=
 honnef.co/go/tools v0.5.1/go.mod h1:e9irvo83WDG9/irijV44wr3tbhcFeRnfpVlRqVwpzMs=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/publishing-bot v0.5.0 h1:Hfnhltr+khEcqvoK4GBYrtaA8dHJ50Xjyi+0KGUfU3I=
+k8s.io/publishing-bot v0.5.0/go.mod h1:S5+zQQhsVUEqdcaohbYf8O+2BeeWRtuYzp4tQLr5An8=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 mvdan.cc/gofumpt v0.6.0 h1:G3QvahNDmpD+Aek/bNOLrFR2XC6ZAdo62dZu65gmwGo=
 mvdan.cc/gofumpt v0.6.0/go.mod h1:4L0wf+kgIPZtcCWXynNS2e6bhmj73umwnuXSZarixzA=

hack/tools/publishing-verifier/publishing-verifier.go

@@ -0,0 +1,240 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/mod/modfile"
+	"k8s.io/publishing-bot/cmd/publishing-bot/config"
+)
+
+var (
+	rulesFile           string
+	componentsDirectory string
+)
+
+// getGoModDependencies gets all the staging dependencies for all the modules
+// in the given directory
+func getGoModDependencies(dir string) (map[string][]string, error) {
+	allDependencies := make(map[string][]string)
+	components, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+	for _, component := range components {
+		componentName := component.Name()
+		if !component.IsDir() {
+			// currently there is no hard check that the staging directory should not contain
+			// other files
+			continue
+		}
+		gomodFilePath := filepath.Join(dir, componentName, "go.mod")
+		gomodFileContent, err := os.ReadFile(gomodFilePath)
+		if err != nil {
+			return nil, err
+		}
+
+		fmt.Printf("%s dependencies", componentName)
+
+		allDependencies[componentName] = make([]string, 0)
+
+		gomodFile, err := modfile.Parse(gomodFilePath, gomodFileContent, nil)
+		if err != nil {
+			return nil, err
+		}
+		// get all the other dependencies from within staging, i.e all the modules in replace
+		// section
+		for _, module := range gomodFile.Replace {
+			dep := strings.TrimPrefix(module.Old.Path, "k8s.io/")
+			if dep == componentName {
+				continue
+			}
+			allDependencies[componentName] = append(allDependencies[componentName], dep)
+		}
+	}
+	return allDependencies, nil
+}
+
+// diffSlice returns the difference of s1-s2
+func diffSlice(s1, s2 []string) []string {
+	var diff []string
+	set := make(map[string]struct{}, len(s2))
+	for _, s := range s2 {
+		set[s] = struct{}{}
+	}
+	for _, s := range s1 {
+		if _, ok := set[s]; !ok {
+			diff = append(diff, s)
+		}
+	}
+	return diff
+}
+
+// getKeys returns a slice with only the keys of the given map
+func getKeys[K comparable, V any](m map[K]V) []K {
+	var keys []K
+	for k := range m {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+// checkValidSourceDirectory checks if proper source directory fields are used in rules
+func checkValidSourceDirectory(rule config.RepositoryRule) error {
+	for _, branch := range rule.Branches {
+		if branch.Source.Dir != "" {
+			return fmt.Errorf("use of deprecated `dir` field in rules for `%s`", rule.DestinationRepository)
+		}
+		if len(branch.Source.Dirs) > 1 {
+			return fmt.Errorf("cannot have more than one directory (%s) per source branch `%s` of `%s`",
+				branch.Source.Dirs,
+				branch.Source.Branch,
+				rule.DestinationRepository,
+			)
+		}
+		if !strings.HasSuffix(branch.Source.Dirs[0], rule.DestinationRepository) {
+			return fmt.Errorf("copy/paste error `%s` refers to `%s`", rule.DestinationRepository, branch.Source.Dirs[0])
+		}
+	}
+	return nil
+}
+
+// checkMasterBranch checks if the master branch of destination repository refers to the master
+// of the source
+func checkMasterBranch(rule config.RepositoryRule) error {
+	branch := rule.Branches[0]
+	if branch.Name != "master" {
+		return fmt.Errorf("cannot find master branch for destination `%s`", rule.DestinationRepository)
+	}
+
+	if branch.Source.Branch != "master" {
+		return fmt.Errorf("cannot find master source branch for destination `%s`", rule.DestinationRepository)
+	}
+	return nil
+}
+
+func checkDependencies(rule config.RepositoryRule, gomodDependencies map[string][]string) error {
+	var processedDeps []string
+	branch := rule.Branches[0]
+	for _, dep := range gomodDependencies[rule.DestinationRepository] {
+		found := false
+		if len(branch.Dependencies) > 0 {
+			for _, dep2 := range branch.Dependencies {
+				processedDeps = append(processedDeps, dep2.Repository)
+				if dep2.Branch != "master" {
+					return fmt.Errorf("looking for master branch of %s and found : %s for destination", dep2.Repository, rule.DestinationRepository)
+				}
+				if dep2.Repository == dep {
+					found = true
+				}
+			}
+		} else {
+			return fmt.Errorf("Please add %s as dependencies under destination %s", gomodDependencies[rule.DestinationRepository], rule.DestinationRepository)
+		}
+		if !found {
+			return fmt.Errorf("Please add %s as a dependency under destination %s", dep, rule.DestinationRepository)
+		} else {
+			fmt.Printf("dependency %s found\n", dep)
+		}
+	}
+	// check if all deps are processed.
+	extraDeps := diffSlice(processedDeps, gomodDependencies[rule.DestinationRepository])
+	if len(extraDeps) > 0 {
+		return fmt.Errorf("extra dependencies in rules for %s: %s", rule.DestinationRepository, strings.Join(extraDeps, ","))
+	}
+	return nil
+}
+
+func verifyPublishingBotRules() error {
+	rules, err := config.LoadRules(rulesFile)
+	if err != nil {
+		return fmt.Errorf("error loading rules: %v", err)
+	}
+
+	gomodDependencies, err := getGoModDependencies(componentsDirectory)
+
+	var processedRepos []string
+	for _, rule := range rules.Rules {
+		branch := rule.Branches[0]
+
+		// if this no longer exists in master
+		if _, ok := gomodDependencies[rule.DestinationRepository]; !ok {
+			// make sure we dont include a rule to publish it from master
+			for _, branch := range rule.Branches {
+				if branch.Name == "master" {
+					err := fmt.Errorf("cannot find master branch for destination `%s`", rule.DestinationRepository)
+					panic(err)
+				}
+			}
+			// and skip the validation of publishing rules for it
+			continue
+		}
+
+		if err := checkValidSourceDirectory(rule); err != nil {
+			return fmt.Errorf("error validating source directory: %v", err)
+		}
+
+		if err := checkMasterBranch(rule); err != nil {
+			return fmt.Errorf("error validating master branch: %v", err)
+		}
+
+		// we specify the go version for all master branches through `default-go-version`
+		// so ensure we don't specify explicit go version for master branch in rules
+		if branch.GoVersion != "" {
+			err := fmt.Errorf("go version must not be specified for master branch for destination `%s`", rule.DestinationRepository)
+			panic(err)
+		}
+
+		fmt.Printf("processing : %s", rule.DestinationRepository)
+		if _, ok := gomodDependencies[rule.DestinationRepository]; !ok {
+			err := fmt.Errorf("missing go.mod for `%s`", rule.DestinationRepository)
+			panic(err)
+		}
+		processedRepos = append(processedRepos, rule.DestinationRepository)
+
+		if err := checkDependencies(rule, gomodDependencies); err != nil {
+			return fmt.Errorf("error validating dependencies: %v", err)
+		}
+	}
+
+	// check if all repos are processed.
+	items := diffSlice(getKeys(gomodDependencies), processedRepos)
+	if len(items) > 0 {
+		err := fmt.Errorf("missing rules for %s", strings.Join(items, ","))
+		panic(err)
+	}
+	return nil
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		panic("invalid number of arguments")
+	}
+
+	kubeRoot := os.Args[1]
+	stagingDirectory := kubeRoot + "/staging/"
+	rulesFile = stagingDirectory + "publishing/rules.yaml"
+	componentsDirectory = stagingDirectory + "src/k8s.io/"
+
+	if err := verifyPublishingBotRules(); err != nil {
+		panic(err)
+	}
+}

hack/tools/tools.go

@@ -36,4 +36,8 @@ import (
 
 	// tools like cpu
 	_ "go.uber.org/automaxprocs"
+
+	// for publishing bot
+	_ "golang.org/x/mod/modfile"
+	_ "k8s.io/publishing-bot/cmd/publishing-bot/config"
 )

hack/verify-publishing-bot.py

@@ -1,134 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import fnmatch
-import os
-import sys
-import json
-
-
-def get_gomod_dependencies(rootdir, components):
-    all_dependencies = {}
-    for component in components:
-        with open(os.path.join(rootdir, component, "go.mod")) as f:
-            print((component + " dependencies"))
-            all_dependencies[component] = []
-            lines = list(set(f))
-            lines.sort()
-            for line in lines:
-                for dep in components:
-                    if dep == component:
-                        continue
-                    if ("k8s.io/" + dep + " =>") not in line:
-                        continue
-                    print(("\t"+dep))
-                    if dep not in all_dependencies[component]:
-                        all_dependencies[component].append(dep)
-    return all_dependencies
-
-
-def get_rules_dependencies(rules_file):
-    import yaml
-    with open(rules_file) as f:
-        data = yaml.safe_load(f)
-    return data
-
-
-def main():
-    rootdir = os.path.dirname(__file__) + "/../"
-    rootdir = os.path.abspath(rootdir)
-
-    components = []
-    for component in os.listdir(rootdir + '/staging/src/k8s.io/'):
-        components.append(component)
-    components.sort()
-
-    rules_file = "/staging/publishing/rules.yaml"
-    try:
-        import yaml
-    except ImportError:
-        print(("Please install missing pyyaml module and re-run %s" % sys.argv[0]))
-        sys.exit(1)
-    rules_dependencies = get_rules_dependencies(rootdir + rules_file)
-
-    gomod_dependencies = get_gomod_dependencies(rootdir + '/staging/src/k8s.io/', components)
-
-    processed_repos = []
-    for rule in rules_dependencies["rules"]:
-        branch = rule["branches"][0]
-
-        # If this no longer exists in master
-        if rule["destination"] not in gomod_dependencies:
-            # Make sure we don't include a rule to publish it from master
-            for branch in rule["branches"]:
-                if branch["name"] == "master":
-                    raise Exception("cannot find master branch for destination %s" % rule["destination"])
-            # And skip validation of publishing rules for it
-            continue
-
-        for item in rule["branches"]:
-            if "dir" in item["source"]:
-                raise Exception("use of deprecated `dir` field in rules for `%s`" % (rule["destination"]))
-            if len(item["source"]["dirs"]) > 1:
-                raise Exception("cannot have more than one directory (`%s`) per source branch `%s` of `%s`" %
-                                (item["source"]["dirs"], item["source"]["branch"], rule["destination"])
-                                )
-            if not item["source"]["dirs"][0].endswith(rule["destination"]):
-                raise Exception("copy/paste error `%s` refers to `%s`" % (rule["destination"],item["source"]["dir"]))
-
-        if branch["name"] != "master":
-            raise Exception("cannot find master branch for destination %s" % rule["destination"])
-        if branch["source"]["branch"] != "master":
-            raise Exception("cannot find master source branch for destination %s" % rule["destination"])
-
-        # we specify the go version for all master branches through `default-go-version`
-        # so ensure we don't specify explicit go version for master branch in rules
-        if "go" in branch:
-            raise Exception("go version must not be specified for master branch for destination %s" % rule["destination"])
-
-        print(("processing : %s" % rule["destination"]))
-        if rule["destination"] not in gomod_dependencies:
-            raise Exception("missing go.mod for %s" % rule["destination"])
-        processed_repos.append(rule["destination"])
-        processed_deps = []
-        for dep in set(gomod_dependencies[rule["destination"]]):
-            found = False
-            if "dependencies" in branch:
-                for dep2 in branch["dependencies"]:
-                    processed_deps.append(dep2["repository"])
-                    if dep2["branch"] != "master":
-                        raise Exception("Looking for master branch and found : %s for destination", dep2,
-                                        rule["destination"])
-                    if dep2["repository"] == dep:
-                        found = True
-            else:
-                raise Exception(
-                    "Please add %s as dependencies under destination %s in %s" % (gomod_dependencies[rule["destination"]], rule["destination"], rules_file))
-            if not found:
-                raise Exception("Please add %s as a dependency under destination %s in %s" % (dep, rule["destination"], rules_file))
-            else:
-                print(("  found dependency %s" % dep))
-        extraDeps = set(processed_deps) - set(gomod_dependencies[rule["destination"]])
-        if len(extraDeps) > 0:
-            raise Exception("extra dependencies in rules for %s: %s" % (rule["destination"], ','.join(str(s) for s in extraDeps)))
-    items = set(gomod_dependencies.keys()) - set(processed_repos)
-    if len(items) > 0:
-        raise Exception("missing rules for %s" % ','.join(str(s) for s in items))
-    print("Done.")
-
-
-if __name__ == "__main__":
-    sys.exit(main())

hack/verify-publishing-bot.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script checks whether staging/publishing/rules.yaml is correct
+# as per the dependencies in the go.mod of the staging directories
+# Usage: `hack/verify-publishing-bot.sh`.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+kube::golang::setup_env
+
+go -C "${KUBE_ROOT}/hack/tools" install ./publishing-verifier
+
+publishing-verifier "${KUBE_ROOT}"