Merge pull request #130648 from jpbetz/semver-tolerant

Enable Semver CEL library, add normalization support

staging/src/k8s.io/apiserver/pkg/cel/environment/base.go

@@ -176,6 +176,13 @@ var baseOptsWithoutStrictCost = []VersionedOptions{
 			ext.TwoVarComprehensions(),
 		},
 	},
+	// Semver
+	{
+		IntroducedVersion: version.MajorMinor(1, 33),
+		EnvOptions: []cel.EnvOption{
+			library.SemverLib(library.SemverVersion(1)),
+		},
+	},
 }
 
 var (

staging/src/k8s.io/apiserver/pkg/cel/library/cost.go

@@ -18,13 +18,14 @@ package library
 
 import (
 	"fmt"
+	"math"
+
 	"github.com/google/cel-go/checker"
 	"github.com/google/cel-go/common"
 	"github.com/google/cel-go/common/ast"
 	"github.com/google/cel-go/common/types"
 	"github.com/google/cel-go/common/types/ref"
 	"github.com/google/cel-go/common/types/traits"
-	"math"
 
 	"k8s.io/apiserver/pkg/cel"
 )
@@ -202,7 +203,7 @@ func (l *CostEstimator) CallCost(function, overloadId string, args []ref.Val, re
 
 			return &cost
 		}
-	case "quantity", "isQuantity":
+	case "quantity", "isQuantity", "semver", "isSemver":
 		if len(args) >= 1 {
 			cost := uint64(math.Ceil(float64(actualSize(args[0])) * common.StringTraversalCostFactor))
 			return &cost
@@ -236,7 +237,7 @@ func (l *CostEstimator) CallCost(function, overloadId string, args []ref.Val, re
 		// Simply dictionary lookup
 		cost := uint64(1)
 		return &cost
-	case "sign", "asInteger", "isInteger", "asApproximateFloat", "isGreaterThan", "isLessThan", "compareTo", "add", "sub":
+	case "sign", "asInteger", "isInteger", "asApproximateFloat", "isGreaterThan", "isLessThan", "compareTo", "add", "sub", "major", "minor", "patch":
 		cost := uint64(1)
 		return &cost
 	case "getScheme", "getHostname", "getHost", "getPort", "getEscapedPath", "getQuery":
@@ -486,7 +487,7 @@ func (l *CostEstimator) EstimateCallCost(function, overloadId string, target *ch
 
 			return &checker.CallEstimate{CostEstimate: ipCompCost}
 		}
-	case "quantity", "isQuantity":
+	case "quantity", "isQuantity", "semver", "isSemver":
 		if target != nil {
 			sz := l.sizeEstimate(args[0])
 			return &checker.CallEstimate{CostEstimate: sz.MultiplyByCostFactor(common.StringTraversalCostFactor)}
@@ -498,7 +499,7 @@ func (l *CostEstimator) EstimateCallCost(function, overloadId string, target *ch
 		}
 	case "format.named":
 		return &checker.CallEstimate{CostEstimate: checker.CostEstimate{Min: 1, Max: 1}}
-	case "sign", "asInteger", "isInteger", "asApproximateFloat", "isGreaterThan", "isLessThan", "compareTo", "add", "sub":
+	case "sign", "asInteger", "isInteger", "asApproximateFloat", "isGreaterThan", "isLessThan", "compareTo", "add", "sub", "major", "minor", "patch":
 		return &checker.CallEstimate{CostEstimate: checker.CostEstimate{Min: 1, Max: 1}}
 	case "getScheme", "getHostname", "getHost", "getPort", "getEscapedPath", "getQuery":
 		// url accessors

staging/src/k8s.io/apiserver/pkg/cel/library/cost_test.go

@@ -19,9 +19,10 @@ package library
 import (
 	"context"
 	"fmt"
-	"github.com/google/cel-go/common/types/ref"
 	"testing"
 
+	"github.com/google/cel-go/common/types/ref"
+
 	"github.com/google/cel-go/cel"
 	"github.com/google/cel-go/checker"
 	"github.com/google/cel-go/common"
@@ -1110,6 +1111,86 @@ func TestSetsCost(t *testing.T) {
 	}
 }
 
+func TestSemverCost(t *testing.T) {
+	cases := []struct {
+		name                string
+		expr                string
+		expectEstimatedCost checker.CostEstimate
+		expectRuntimeCost   uint64
+	}{
+		{
+			name:                "semver",
+			expr:                `semver("1.0.0")`,
+			expectEstimatedCost: checker.CostEstimate{Min: 1, Max: 1},
+			expectRuntimeCost:   1,
+		},
+		{
+			name:                "semver long input",
+			expr:                `semver("1234.56789012345.67890123456789")`,
+			expectEstimatedCost: checker.CostEstimate{Min: 4, Max: 4},
+			expectRuntimeCost:   4,
+		},
+		{
+			name:                "isSemver",
+			expr:                `isSemver("1.0.0")`,
+			expectEstimatedCost: checker.CostEstimate{Min: 1, Max: 1},
+			expectRuntimeCost:   1,
+		},
+		{
+			name:                "isSemver long input",
+			expr:                `isSemver("1234.56789012345.67890123456789")`,
+			expectEstimatedCost: checker.CostEstimate{Min: 4, Max: 4},
+			expectRuntimeCost:   4,
+		},
+		// major(), minor(), patch()
+		{
+			name:                "major",
+			expr:                `semver("1.2.3").major()`,
+			expectEstimatedCost: checker.CostEstimate{Min: 2, Max: 2},
+			expectRuntimeCost:   2,
+		},
+		{
+			name:                "minor",
+			expr:                `semver("1.2.3").minor()`,
+			expectEstimatedCost: checker.CostEstimate{Min: 2, Max: 2},
+			expectRuntimeCost:   2,
+		},
+		{
+			name:                "patch",
+			expr:                `semver("1.2.3").patch()`,
+			expectEstimatedCost: checker.CostEstimate{Min: 2, Max: 2},
+			expectRuntimeCost:   2,
+		},
+		// isLessThan
+		{
+			name:                "isLessThan",
+			expr:                `semver("1.0.0").isLessThan(semver("1.1.0"))`,
+			expectEstimatedCost: checker.CostEstimate{Min: 3, Max: 3},
+			expectRuntimeCost:   3,
+		},
+		// isGreaterThan
+		{
+			name:                "isGreaterThan",
+			expr:                `semver("1.1.0").isGreaterThan(semver("1.0.0"))`,
+			expectEstimatedCost: checker.CostEstimate{Min: 3, Max: 3},
+			expectRuntimeCost:   3,
+		},
+		// compareTo
+		{
+			name:                "compareTo",
+			expr:                `semver("1.0.0").compareTo(semver("1.2.3"))`,
+			expectEstimatedCost: checker.CostEstimate{Min: 3, Max: 3},
+			expectRuntimeCost:   3,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			testCost(t, tc.expr, tc.expectEstimatedCost, tc.expectRuntimeCost)
+		})
+	}
+}
+
 func TestTwoVariableComprehensionCost(t *testing.T) {
 	cases := []struct {
 		name                string
@@ -1223,6 +1304,7 @@ func testCost(t *testing.T, expr string, expectEsimatedCost checker.CostEstimate
 		// Previous the presence has a cost of 0 but cel fixed it to 1. We still set to 0 here to avoid breaking changes.
 		cel.CostEstimatorOptions(checker.PresenceTestHasCost(false)),
 		ext.TwoVarComprehensions(),
+		SemverLib(SemverVersion(1)),
 	)
 	if err != nil {
 		t.Fatalf("%v", err)

staging/src/k8s.io/apiserver/pkg/cel/library/library_compatibility_test.go

@@ -17,11 +17,12 @@ limitations under the License.
 package library
 
 import (
+	"strings"
+	"testing"
+
 	"github.com/google/cel-go/cel"
 	"github.com/google/cel-go/common/decls"
 	"github.com/google/cel-go/common/types"
-	"strings"
-	"testing"
 
 	"k8s.io/apimachinery/pkg/util/sets"
 )
@@ -56,6 +57,8 @@ func TestLibraryCompatibility(t *testing.T) {
 		"fieldSelector", "labelSelector", "validate", "format.named", "isSemver", "major", "minor", "patch", "semver",
 		// Kubernetes <1.32>:
 		"jsonpatch.escapeKey",
+		// Kubernetes <1.33>:
+		"semver", "isSemver", "major", "minor", "patch",
 		// Kubernetes <1.??>:
 	)
 

staging/src/k8s.io/apiserver/pkg/cel/library/semver_test.go

@@ -31,9 +31,9 @@ import (
 	library "k8s.io/apiserver/pkg/cel/library"
 )
 
-func testSemver(t *testing.T, expr string, expectResult ref.Val, expectRuntimeErrPattern string, expectCompileErrs []string) {
+func testSemver(t *testing.T, expr string, expectResult ref.Val, expectRuntimeErrPattern string, expectCompileErrs []string, version uint32) {
 	env, err := cel.NewEnv(
-		library.SemverLib(),
+		library.SemverLib(library.SemverVersion(version)),
 	)
 	if err != nil {
 		t.Fatalf("%v", err)
@@ -114,6 +114,7 @@ func TestSemver(t *testing.T) {
 		expectValue        ref.Val
 		expectedCompileErr []string
 		expectedRuntimeErr string
+		version            uint32
 	}{
 		{
 			name:        "parse",
@@ -131,15 +132,104 @@ func TestSemver(t *testing.T) {
 			expectValue: trueVal,
 		},
 		{
-			name:        "isSemver_false",
+			name:        "isSemver_empty_false",
-			expr:        `isSemver("v1.0")`,
+			expr:        `isSemver("")`,
 			expectValue: falseVal,
 		},
+		{
+			name:        "isSemver_v_prefix_false",
+			expr:        `isSemver("v1.0.0")`,
+			expectValue: falseVal,
+		},
+		{
+			name:        "isSemver_v_leading_whitespace_false",
+			expr:        `isSemver(" 1.0.0")`,
+			expectValue: falseVal,
+		},
+		{
+			name:        "isSemver_v_contains_whitespace_false",
+			expr:        `isSemver("1. 0.0")`,
+			expectValue: falseVal,
+		},
+		{
+			name:        "isSemver_v_trailing_whitespace_false",
+			expr:        `isSemver("1.0.0 ")`,
+			expectValue: falseVal,
+		},
+		{
+			name:        "isSemver_leading_zeros_false",
+			expr:        `isSemver("01.01.01")`,
+			expectValue: falseVal,
+		},
+		{
+			name:        "isSemver_major_only_false",
+			expr:        `isSemver("1")`,
+			expectValue: falseVal,
+		},
+		{
+			name:        "isSemver_major_minor_only_false",
+			expr:        `isSemver("1.1")`,
+			expectValue: falseVal,
+		},
+		{
+			name:        "isSemver_empty_normalize_false",
+			expr:        `isSemver("", true)`,
+			expectValue: falseVal,
+			version:     1,
+		},
+		{
+			name:        "isSemver_v_leading_whitespace_normalize_false",
+			expr:        `isSemver(" 1.0.0", true)`,
+			expectValue: falseVal,
+			version:     1,
+		},
+		{
+			name:        "isSemver_v_contains_whitespace_normalize_false",
+			expr:        `isSemver("1. 0.0", true)`,
+			expectValue: falseVal,
+			version:     1,
+		},
+		{
+			name:        "isSemver_v_trailing_whitespace_normalize_false",
+			expr:        `isSemver("1.0.0 ", true)`,
+			expectValue: falseVal,
+			version:     1,
+		},
+		{
+			name:        "isSemver_v_prefix_normalize_true",
+			expr:        `isSemver("v1.0.0", true)`,
+			expectValue: trueVal,
+			version:     1,
+		},
+		{
+			name:        "isSemver_leading_zeros_normalize_true",
+			expr:        `isSemver("01.01.01", true)`,
+			expectValue: trueVal,
+			version:     1,
+		},
+		{
+			name:        "isSemver_major_only_normalize_true",
+			expr:        `isSemver("1", true)`,
+			expectValue: trueVal,
+			version:     1,
+		},
+		{
+			name:        "isSemver_major_minor_only_normalize_true",
+			expr:        `isSemver("1.1", true)`,
+			expectValue: trueVal,
+			version:     1,
+		},
 		{
 			name:               "isSemver_noOverload",
 			expr:               `isSemver([1, 2, 3])`,
 			expectedCompileErr: []string{"found no matching overload for 'isSemver' applied to.*"},
 		},
+		{
+			name:        "equality_normalize",
+			expr:        `semver("v01.01", true) == semver("1.1.0")`,
+			expectValue: trueVal,
+			version:     1,
+		},
 		{
 			name:        "equality_reflexivity",
 			expr:        `semver("1.2.3") == semver("1.2.3")`,
@@ -204,7 +294,7 @@ func TestSemver(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			testSemver(t, c.expr, c.expectValue, c.expectedRuntimeErr, c.expectedCompileErr)
+			testSemver(t, c.expr, c.expectValue, c.expectedRuntimeErr, c.expectedCompileErr, c.version)
 		})
 	}
 }

staging/src/k8s.io/apiserver/pkg/cel/library/semverlib.go

@@ -17,6 +17,10 @@ limitations under the License.
 package library
 
 import (
+	"errors"
+	"math"
+	"strings"
+
 	"github.com/blang/semver/v4"
 	"github.com/google/cel-go/cel"
 	"github.com/google/cel-go/common/types"
@@ -31,8 +35,10 @@ import (
 //
 // Converts a string to a semantic version or results in an error if the string is not a valid semantic version. Refer
 // to semver.org documentation for information on accepted patterns.
-//
+// An optional "normalize" argument can be passed to enable normalization. Normalization removes any "v" prefix, adds a
+// 0 minor and patch numbers to versions with only major or major.minor components specified, and removes any leading 0s.
 //	semver(<string>) <Semver>
+//	semver(<string>, <bool>) <Semver>
 //
 // Examples:
 //
@@ -41,19 +47,28 @@ import (
 //	semver('200K') // error
 //	semver('Three') // error
 //	semver('Mi') // error
+//	semver('v1.0.0', true) // Applies normalization to remove the leading "v". Returns a Semver of "1.0.0".
+//	semver('1.0', true) // Applies normalization to add the missing patch version. Returns a Semver of "1.0.0"
+//	semver('01.01.01', true) // Applies normalization to remove leading zeros. Returns a Semver of "1.1.1"
 //
 // isSemver
 //
 // Returns true if a string is a valid Semver. isSemver returns true if and
 // only if semver does not result in error.
+// An optional "normalize" argument can be passed to enable normalization. Normalization removes any "v" prefix, adds a
+// 0 minor and patch numbers to versions with only major or major.minor components specified, and removes any leading 0s.
 //
 //	isSemver( <string>) <bool>
+//	isSemver( <string>, <bool>) <bool>
 //
 // Examples:
 //
 //	isSemver('1.0.0') // returns true
-//	isSemver('v1.0') // returns true (tolerant parsing)
 //	isSemver('hello') // returns false
+//  isSemver('v1.0')  // returns false (leading "v" is not allowed unless normalization is enabled)
+//	isSemver('v1.0', true) // Applies normalization to remove leading "v". returns true
+//	semver('1.0', true) // Applies normalization to add the missing patch version. Returns true
+//	semver('01.01.01', true) // Applies normalization to remove leading zeros. Returns true
 //
 // Conversion to Scalars:
 //
@@ -84,13 +99,29 @@ import (
 // semver("1.2.3").compareTo(semver("2.0.0")) // returns -1
 // semver("1.2.3").compareTo(semver("0.1.2")) // returns 1
 
-func SemverLib() cel.EnvOption {
+func SemverLib(options ...SemverOption) cel.EnvOption {
+	semverLib := &semverLibType{}
+	for _, o := range options {
+		semverLib = o(semverLib)
+	}
 	return cel.Lib(semverLib)
 }
 
-var semverLib = &semverLibType{}
+var semverLib = &semverLibType{version: math.MaxUint32} // include all versions
 
-type semverLibType struct{}
+type semverLibType struct {
+	version uint32
+}
+
+// StringsOption is a functional interface for configuring the strings library.
+type SemverOption func(*semverLibType) *semverLibType
+
+func SemverVersion(version uint32) SemverOption {
+	return func(lib *semverLibType) *semverLibType {
+		lib.version = version
+		return lib
+	}
+}
 
 func (*semverLibType) LibraryName() string {
 	return "kubernetes.Semver"
@@ -100,8 +131,8 @@ func (*semverLibType) Types() []*cel.Type {
 	return []*cel.Type{apiservercel.SemverType}
 }
 
-func (*semverLibType) declarations() map[string][]cel.FunctionOpt {
+func (lib *semverLibType) declarations() map[string][]cel.FunctionOpt {
-	return map[string][]cel.FunctionOpt{
+	fnOpts := map[string][]cel.FunctionOpt{
 		"semver": {
 			cel.Overload("string_to_semver", []*cel.Type{cel.StringType}, apiservercel.SemverType, cel.UnaryBinding((stringToSemver))),
 		},
@@ -127,6 +158,11 @@ func (*semverLibType) declarations() map[string][]cel.FunctionOpt {
 			cel.MemberOverload("semver_patch", []*cel.Type{apiservercel.SemverType}, cel.IntType, cel.UnaryBinding(semverPatch)),
 		},
 	}
+	if lib.version >= 1 {
+		fnOpts["semver"] = append(fnOpts["semver"], cel.Overload("string_bool_to_semver", []*cel.Type{cel.StringType, cel.BoolType}, apiservercel.SemverType, cel.BinaryBinding((stringToSemverNormalize))))
+		fnOpts["isSemver"] = append(fnOpts["isSemver"], cel.Overload("is_semver_string_bool", []*cel.Type{cel.StringType, cel.BoolType}, cel.BoolType, cel.BinaryBinding(isSemverNormalize)))
+	}
+	return fnOpts
 }
 
 func (s *semverLibType) CompileOptions() []cel.EnvOption {
@@ -144,16 +180,29 @@ func (*semverLibType) ProgramOptions() []cel.ProgramOption {
 }
 
 func isSemver(arg ref.Val) ref.Val {
+	return isSemverNormalize(arg, types.Bool(false))
+}
+func isSemverNormalize(arg ref.Val, normalizeArg ref.Val) ref.Val {
 	str, ok := arg.Value().(string)
 	if !ok {
 		return types.MaybeNoSuchOverloadErr(arg)
 	}
 
+	normalize, ok := normalizeArg.Value().(bool)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
 	// Using semver/v4 here is okay because this function isn't
 	// used to validate the Kubernetes API. In the CEL base library
 	// we would have to use the regular expression from
 	// pkg/apis/resource/structured/namedresources/validation/validation.go.
-	_, err := semver.Parse(str)
+	var err error
+	if normalize {
+		_, err = normalizeAndParse(str)
+	} else {
+		_, err = semver.Parse(str)
+	}
 	if err != nil {
 		return types.Bool(false)
 	}
@@ -162,17 +211,31 @@ func isSemver(arg ref.Val) ref.Val {
 }
 
 func stringToSemver(arg ref.Val) ref.Val {
+	return stringToSemverNormalize(arg, types.Bool(false))
+}
+func stringToSemverNormalize(arg ref.Val, normalizeArg ref.Val) ref.Val {
 	str, ok := arg.Value().(string)
 	if !ok {
 		return types.MaybeNoSuchOverloadErr(arg)
 	}
 
+	normalize, ok := normalizeArg.Value().(bool)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
 	// Using semver/v4 here is okay because this function isn't
 	// used to validate the Kubernetes API. In the CEL base library
 	// we would have to use the regular expression from
 	// pkg/apis/resource/structured/namedresources/validation/validation.go
 	// first before parsing.
-	v, err := semver.Parse(str)
+	var err error
+	var v semver.Version
+	if normalize {
+		v, err = normalizeAndParse(str)
+	} else {
+		v, err = semver.Parse(str)
+	}
 	if err != nil {
 		return types.WrapErr(err)
 	}
@@ -245,3 +308,37 @@ func semverCompareTo(arg ref.Val, other ref.Val) ref.Val {
 
 	return types.Int(v.Compare(v2))
 }
+
+// normalizeAndParse removes any "v" prefix,  adds a 0 minor and patch numbers to versions with
+// only major or major.minor components specified, and removes any leading 0s.
+// normalizeAndParse is based on semver.ParseTolerant but does not trim extra whitespace and is
+// guaranteed to not change behavior in the future.
+func normalizeAndParse(s string) (semver.Version, error) {
+	s = strings.TrimPrefix(s, "v")
+
+	// Split into major.minor.(patch+pr+meta)
+	parts := strings.SplitN(s, ".", 3)
+	// Remove leading zeros.
+	for i, p := range parts {
+		if len(p) > 1 {
+			p = strings.TrimLeft(p, "0")
+			if len(p) == 0 || !strings.ContainsAny(p[0:1], "0123456789") {
+				p = "0" + p
+			}
+			parts[i] = p
+		}
+	}
+
+	// Fill up shortened versions.
+	if len(parts) < 3 {
+		if strings.ContainsAny(parts[len(parts)-1], "+-") {
+			return semver.Version{}, errors.New("short version cannot contain PreRelease/Build meta data")
+		}
+		for len(parts) < 3 {
+			parts = append(parts, "0")
+		}
+	}
+	s = strings.Join(parts, ".")
+
+	return semver.Parse(s)
+}

staging/src/k8s.io/dynamic-resource-allocation/cel/compile.go

@@ -33,13 +33,14 @@ import (
 	"github.com/google/cel-go/common/types/traits"
 	"github.com/google/cel-go/ext"
 
+	"k8s.io/utils/ptr"
+
 	resourceapi "k8s.io/api/resource/v1beta1"
 	"k8s.io/apimachinery/pkg/util/version"
 	celconfig "k8s.io/apiserver/pkg/apis/cel"
 	apiservercel "k8s.io/apiserver/pkg/cel"
 	"k8s.io/apiserver/pkg/cel/environment"
 	"k8s.io/apiserver/pkg/cel/library"
-	"k8s.io/utils/ptr"
 )
 
 const (
@@ -297,8 +298,6 @@ func newCompiler() *compiler {
 			EnvOptions: []cel.EnvOption{
 				cel.Variable(deviceVar, deviceType.CelType()),
 
-				environment.UnversionedLib(library.SemverLib),
-
 				// https://pkg.go.dev/github.com/google/cel-go/ext#Bindings
 				//
 				// This is useful to simplify attribute lookups because the
@@ -311,6 +310,22 @@ func newCompiler() *compiler {
 				deviceType,
 			},
 		},
+		{
+			IntroducedVersion: version.MajorMinor(1, 31),
+			// This library has added to base environment of Kubernetes
+			// in 1.33 at version 1. It will continue to be available for
+			// use in this environment, but does not need to be included
+			// directly since it becomes available indirectly via the base
+			// environment shared across Kubernetes.
+			// In Kubernetes 1.34, version 1 feature of this library will
+			// become available, and will be rollback safe to 1.33.
+			// TODO: In Kubernetes 1.34: Add compile tests that demonstrate that
+			// `isSemver("v1.0.0", true)` and `semver("v1.0.0", true)` are supported.
+			RemovedVersion: version.MajorMinor(1, 33),
+			EnvOptions: []cel.EnvOption{
+				library.SemverLib(library.SemverVersion(0)),
+			},
+		},
 	}
 	envset, err := envset.Extend(versioned...)
 	if err != nil {