scottk/kubernetes
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/kubernetes.git synced 2025-10-31 02:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

rewrite publishing-bot verify script in go

Browse Source 
- setup scripts to trigger publishing-bot verify
- add golang verify script and remove python verification code
- pin k8s.io/publishing-bot to v0.5.0

Signed-off-by: Akhil Mohan <akhilerm@gmail.com>
This commit is contained in:
Akhil Mohan
2024-06-29 22:06:53 +05:30
parent 1dbf0830c5
commit b0bf3fba54
6 changed files with 285 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hack/tools/go.mod
View File

@@ -10,8 +10,10 @@ require (
github.com/jcchavezs/porto v0.6.0
github.com/vektra/mockery/v2 v2.40.3
go.uber.org/automaxprocs v1.5.2
golang.org/x/mod v0.20.0
gotest.tools/gotestsum v1.12.0
honnef.co/go/tools v0.5.1
k8s.io/publishing-bot v0.5.0
sigs.k8s.io/logtools v0.8.1
)
@@ -74,6 +76,7 @@ require (
github.com/go-xmlfmt/xmlfmt v1.1.2 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/golang/glog v1.2.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
@@ -190,7 +193,6 @@ require (
go.uber.org/zap v1.24.0 // indirect
golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
golang.org/x/exp/typeparams v0.0.0-20231219180239-dc181d75b848 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/term v0.18.0 // indirect

8
hack/tools/go.sum
View File

@@ -207,6 +207,8 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY=
github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -678,8 +680,8 @@ golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1024,6 +1026,8 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
honnef.co/go/tools v0.5.1 h1:4bH5o3b5ZULQ4UrBmP+63W9r7qIkqJClEA9ko5YKx+I=
honnef.co/go/tools v0.5.1/go.mod h1:e9irvo83WDG9/irijV44wr3tbhcFeRnfpVlRqVwpzMs=
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/publishing-bot v0.5.0 h1:Hfnhltr+khEcqvoK4GBYrtaA8dHJ50Xjyi+0KGUfU3I=
k8s.io/publishing-bot v0.5.0/go.mod h1:S5+zQQhsVUEqdcaohbYf8O+2BeeWRtuYzp4tQLr5An8=
k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
mvdan.cc/gofumpt v0.6.0 h1:G3QvahNDmpD+Aek/bNOLrFR2XC6ZAdo62dZu65gmwGo=
mvdan.cc/gofumpt v0.6.0/go.mod h1:4L0wf+kgIPZtcCWXynNS2e6bhmj73umwnuXSZarixzA=

240
hack/tools/publishing-verifier/publishing-verifier.go Normal file
View File

@@ -0,0 +1,240 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"golang.org/x/mod/modfile"
"k8s.io/publishing-bot/cmd/publishing-bot/config"
)
var (
rulesFile string
componentsDirectory string
)
// getGoModDependencies gets all the staging dependencies for all the modules
// in the given directory
func getGoModDependencies(dir string) (map[string][]string, error) {
allDependencies := make(map[string][]string)
components, err := os.ReadDir(dir)
if err != nil {
return nil, err
}
for _, component := range components {
componentName := component.Name()
if !component.IsDir() {
// currently there is no hard check that the staging directory should not contain
// other files
continue
}
gomodFilePath := filepath.Join(dir, componentName, "go.mod")
gomodFileContent, err := os.ReadFile(gomodFilePath)
if err != nil {
return nil, err
}
fmt.Printf("%s dependencies", componentName)
allDependencies[componentName] = make([]string, 0)
gomodFile, err := modfile.Parse(gomodFilePath, gomodFileContent, nil)
if err != nil {
return nil, err
}
// get all the other dependencies from within staging, i.e all the modules in replace
// section
for _, module := range gomodFile.Replace {
dep := strings.TrimPrefix(module.Old.Path, "k8s.io/")
if dep == componentName {
continue
}
allDependencies[componentName] = append(allDependencies[componentName], dep)
}
}
return allDependencies, nil
}
// diffSlice returns the difference of s1-s2
func diffSlice(s1, s2 []string) []string {
var diff []string
set := make(map[string]struct{}, len(s2))
for _, s := range s2 {
set[s] = struct{}{}
}
for _, s := range s1 {
if _, ok := set[s]; !ok {
diff = append(diff, s)
}
}
return diff
}
// getKeys returns a slice with only the keys of the given map
func getKeys[K comparable, V any](m map[K]V) []K {
var keys []K
for k := range m {
keys = append(keys, k)
}
return keys
}
// checkValidSourceDirectory checks if proper source directory fields are used in rules
func checkValidSourceDirectory(rule config.RepositoryRule) error {
for _, branch := range rule.Branches {
if branch.Source.Dir != "" {
return fmt.Errorf("use of deprecated `dir` field in rules for `%s`", rule.DestinationRepository)
}
if len(branch.Source.Dirs) > 1 {
return fmt.Errorf("cannot have more than one directory (%s) per source branch `%s` of `%s`",
branch.Source.Dirs,
branch.Source.Branch,
rule.DestinationRepository,
)
}
if !strings.HasSuffix(branch.Source.Dirs[0], rule.DestinationRepository) {
return fmt.Errorf("copy/paste error `%s` refers to `%s`", rule.DestinationRepository, branch.Source.Dirs[0])
}
}
return nil
}
// checkMasterBranch checks if the master branch of destination repository refers to the master
// of the source
func checkMasterBranch(rule config.RepositoryRule) error {
branch := rule.Branches[0]
if branch.Name != "master" {
return fmt.Errorf("cannot find master branch for destination `%s`", rule.DestinationRepository)
}
if branch.Source.Branch != "master" {
return fmt.Errorf("cannot find master source branch for destination `%s`", rule.DestinationRepository)
}
return nil
}
func checkDependencies(rule config.RepositoryRule, gomodDependencies map[string][]string) error {
var processedDeps []string
branch := rule.Branches[0]
for _, dep := range gomodDependencies[rule.DestinationRepository] {
found := false
if len(branch.Dependencies) > 0 {
for _, dep2 := range branch.Dependencies {
processedDeps = append(processedDeps, dep2.Repository)
if dep2.Branch != "master" {
return fmt.Errorf("looking for master branch of %s and found : %s for destination", dep2.Repository, rule.DestinationRepository)
}
if dep2.Repository == dep {
found = true
}
}
} else {
return fmt.Errorf("Please add %s as dependencies under destination %s", gomodDependencies[rule.DestinationRepository], rule.DestinationRepository)
}
if !found {
return fmt.Errorf("Please add %s as a dependency under destination %s", dep, rule.DestinationRepository)
} else {
fmt.Printf("dependency %s found\n", dep)
}
}
// check if all deps are processed.
extraDeps := diffSlice(processedDeps, gomodDependencies[rule.DestinationRepository])
if len(extraDeps) > 0 {
return fmt.Errorf("extra dependencies in rules for %s: %s", rule.DestinationRepository, strings.Join(extraDeps, ","))
}
return nil
}
func verifyPublishingBotRules() error {
rules, err := config.LoadRules(rulesFile)
if err != nil {
return fmt.Errorf("error loading rules: %v", err)
}
gomodDependencies, err := getGoModDependencies(componentsDirectory)
var processedRepos []string
for _, rule := range rules.Rules {
branch := rule.Branches[0]
// if this no longer exists in master
if _, ok := gomodDependencies[rule.DestinationRepository]; !ok {
// make sure we dont include a rule to publish it from master
for _, branch := range rule.Branches {
if branch.Name == "master" {
err := fmt.Errorf("cannot find master branch for destination `%s`", rule.DestinationRepository)
panic(err)
}
}
// and skip the validation of publishing rules for it
continue
}
if err := checkValidSourceDirectory(rule); err != nil {
return fmt.Errorf("error validating source directory: %v", err)
}
if err := checkMasterBranch(rule); err != nil {
return fmt.Errorf("error validating master branch: %v", err)
}
// we specify the go version for all master branches through `default-go-version`
// so ensure we don't specify explicit go version for master branch in rules
if branch.GoVersion != "" {
err := fmt.Errorf("go version must not be specified for master branch for destination `%s`", rule.DestinationRepository)
panic(err)
}
fmt.Printf("processing : %s", rule.DestinationRepository)
if _, ok := gomodDependencies[rule.DestinationRepository]; !ok {
err := fmt.Errorf("missing go.mod for `%s`", rule.DestinationRepository)
panic(err)
}
processedRepos = append(processedRepos, rule.DestinationRepository)
if err := checkDependencies(rule, gomodDependencies); err != nil {
return fmt.Errorf("error validating dependencies: %v", err)
}
}
// check if all repos are processed.
items := diffSlice(getKeys(gomodDependencies), processedRepos)
if len(items) > 0 {
err := fmt.Errorf("missing rules for %s", strings.Join(items, ","))
panic(err)
}
return nil
}
func main() {
if len(os.Args) != 2 {
panic("invalid number of arguments")
}
kubeRoot := os.Args[1]
stagingDirectory := kubeRoot + "/staging/"
rulesFile = stagingDirectory + "publishing/rules.yaml"
componentsDirectory = stagingDirectory + "src/k8s.io/"
if err := verifyPublishingBotRules(); err != nil {
panic(err)
}
}

4
hack/tools/tools.go
View File

@@ -36,4 +36,8 @@ import (
// tools like cpu
_ "go.uber.org/automaxprocs"
// for publishing bot
_ "golang.org/x/mod/modfile"
_ "k8s.io/publishing-bot/cmd/publishing-bot/config"
)

134
hack/verify-publishing-bot.py
View File

@@ -1,134 +0,0 @@
#!/usr/bin/env python3
# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import fnmatch
import os
import sys
import json
def get_gomod_dependencies(rootdir, components):
all_dependencies = {}
for component in components:
with open(os.path.join(rootdir, component, "go.mod")) as f:
print((component + " dependencies"))
all_dependencies[component] = []
lines = list(set(f))
lines.sort()
for line in lines:
for dep in components:
if dep == component:
continue
if ("k8s.io/" + dep + " =>") not in line:
continue
print(("\t"+dep))
if dep not in all_dependencies[component]:
all_dependencies[component].append(dep)
return all_dependencies
def get_rules_dependencies(rules_file):
import yaml
with open(rules_file) as f:
data = yaml.safe_load(f)
return data
def main():
rootdir = os.path.dirname(__file__) + "/../"
rootdir = os.path.abspath(rootdir)
components = []
for component in os.listdir(rootdir + '/staging/src/k8s.io/'):
components.append(component)
components.sort()
rules_file = "/staging/publishing/rules.yaml"
try:
import yaml
except ImportError:
print(("Please install missing pyyaml module and re-run %s" % sys.argv[0]))
sys.exit(1)
rules_dependencies = get_rules_dependencies(rootdir + rules_file)
gomod_dependencies = get_gomod_dependencies(rootdir + '/staging/src/k8s.io/', components)
processed_repos = []
for rule in rules_dependencies["rules"]:
branch = rule["branches"][0]
# If this no longer exists in master
if rule["destination"] not in gomod_dependencies:
# Make sure we don't include a rule to publish it from master
for branch in rule["branches"]:
if branch["name"] == "master":
raise Exception("cannot find master branch for destination %s" % rule["destination"])
# And skip validation of publishing rules for it
continue
for item in rule["branches"]:
if "dir" in item["source"]:
raise Exception("use of deprecated `dir` field in rules for `%s`" % (rule["destination"]))
if len(item["source"]["dirs"]) > 1:
raise Exception("cannot have more than one directory (`%s`) per source branch `%s` of `%s`" %
(item["source"]["dirs"], item["source"]["branch"], rule["destination"])
)
if not item["source"]["dirs"][0].endswith(rule["destination"]):
raise Exception("copy/paste error `%s` refers to `%s`" % (rule["destination"],item["source"]["dir"]))
if branch["name"] != "master":
raise Exception("cannot find master branch for destination %s" % rule["destination"])
if branch["source"]["branch"] != "master":
raise Exception("cannot find master source branch for destination %s" % rule["destination"])
# we specify the go version for all master branches through `default-go-version`
# so ensure we don't specify explicit go version for master branch in rules
if "go" in branch:
raise Exception("go version must not be specified for master branch for destination %s" % rule["destination"])
print(("processing : %s" % rule["destination"]))
if rule["destination"] not in gomod_dependencies:
raise Exception("missing go.mod for %s" % rule["destination"])
processed_repos.append(rule["destination"])
processed_deps = []
for dep in set(gomod_dependencies[rule["destination"]]):
found = False
if "dependencies" in branch:
for dep2 in branch["dependencies"]:
processed_deps.append(dep2["repository"])
if dep2["branch"] != "master":
raise Exception("Looking for master branch and found : %s for destination", dep2,
rule["destination"])
if dep2["repository"] == dep:
found = True
else:
raise Exception(
"Please add %s as dependencies under destination %s in %s" % (gomod_dependencies[rule["destination"]], rule["destination"], rules_file))
if not found:
raise Exception("Please add %s as a dependency under destination %s in %s" % (dep, rule["destination"], rules_file))
else:
print((" found dependency %s" % dep))
extraDeps = set(processed_deps) - set(gomod_dependencies[rule["destination"]])
if len(extraDeps) > 0:
raise Exception("extra dependencies in rules for %s: %s" % (rule["destination"], ','.join(str(s) for s in extraDeps)))
items = set(gomod_dependencies.keys()) - set(processed_repos)
if len(items) > 0:
raise Exception("missing rules for %s" % ','.join(str(s) for s in items))
print("Done.")
if __name__ == "__main__":
sys.exit(main())

32
hack/verify-publishing-bot.sh Executable file
View File

@@ -0,0 +1,32 @@
#!/usr/bin/env bash
# Copyright 2024 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script checks whether staging/publishing/rules.yaml is correct
# as per the dependencies in the go.mod of the staging directories
# Usage: `hack/verify-publishing-bot.sh`.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
kube::golang::setup_env
go -C "${KUBE_ROOT}/hack/tools" install ./publishing-verifier
publishing-verifier "${KUBE_ROOT}"