scottk/kubernetes
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/kubernetes.git synced 2025-11-03 19:58:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use json-iterator for JSON, kill off codecgen

Browse Source
This commit is contained in:
Tim Hockin
2017-08-29 17:32:35 -07:00
parent 58aa139f99
commit 5728b1970a
10 changed files with 53 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
hack/godep-save.sh
View File

@@ -34,7 +34,6 @@ fi
# Some things we want in godeps aren't code dependencies, so ./...
# won't pick them up.
REQUIRED_BINS=(
"github.com/ugorji/go/codec/codecgen"
"github.com/onsi/ginkgo/ginkgo"
"github.com/jteeuwen/go-bindata/go-bindata"
"./..."

2
hack/lib/golang.sh
View File

@@ -383,7 +383,7 @@ kube::golang::setup_env() {
# Unset GOBIN in case it already exists in the current session.
unset GOBIN
# This seems to matter to some tools (godep, ugorji, ginkgo...)
# This seems to matter to some tools (godep, ginkgo...)
export GO15VENDOREXPERIMENT=1
}

1
hack/update-all.sh
View File

@@ -63,7 +63,6 @@ fi
BASH_TARGETS="
update-generated-protobuf
update-codegen
update-codecgen
update-generated-docs
update-generated-swagger-docs
update-swagger-spec

176
hack/update-codecgen.sh
View File

@@ -1,176 +0,0 @@
#!/bin/bash
# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
kube::golang::setup_env
# The sort at the end makes sure we feed the topological sort a deterministic
# list (since there aren't many dependencies).
generated_files=($(
find . -not \( \
\( \
-wholename './output' \
-o -wholename './_output' \
-o -wholename './staging' \
-o -wholename './release' \
-o -wholename './target' \
-o -wholename '*/third_party/*' \
-o -wholename '*/vendor/*' \
-o -wholename '*/codecgen-*-1234.generated.go' \
\) -prune \
\) -name '*.generated.go' | LC_ALL=C sort -r
find ./staging/src -not \( \
\( \
-wholename './output' \
-o -wholename './_output' \
-o -wholename './release' \
-o -wholename './target' \
-o -wholename '*/third_party/*' \
-o -wholename '*/codecgen-*-1234.generated.go' \
\) -prune \
\) -name '*.generated.go' | sed 's,staging/src,vendor,' | LC_ALL=C sort -r
))
number=${#generated_files[@]}
###for f in $(echo "${generated_files[@]}" | LC_ALL=C sort); do
### echo "DBG: generated_files: $f"
###done
function package () {
dirname "${1}" | sed 's,\./,k8s.io/kubernetes/,'
}
# extract package list we care about.
dep_packages=($(
for f in "${generated_files[@]}"; do
package "${f}"
done | LC_ALL=C sort -u
))
###for f in $(echo "${dep_packages[@]}"); do
### echo "DBG: dep_packages: $f"
###done
# Register function to be called on EXIT to remove codecgen
# binary and also to touch the files that should be regenerated
# since they are first removed.
# This is necessary to make the script work after previous failure.
function cleanup {
rm -f "${CODECGEN:-}"
pushd "${KUBE_ROOT}" > /dev/null
for f in "${generated_files[@]}"; do
touch "${f}" || true
done
popd > /dev/null
}
trap cleanup EXIT
# Precompute dependencies for all directories.
# Then sort all files in the dependency order.
result=""
dep_packages_grep_pattern=$(echo ${dep_packages[@]} | tr " " '|')
###echo "DBG: dep_packages_grep_pattern: ${dep_packages_grep_pattern}"
for (( i=0; i<number; i++ )); do
visited[${i}]=false
file="${generated_files[${i}]/\.generated\.go/.go}"
deps[${i}]=$(go list -f '{{range .Deps}}{{.}}{{"\n"}}{{end}}' ${file} | grep -Eow "^\\(${dep_packages_grep_pattern}\\)\$" | tr $"\n" ' ' || true)
###echo "DBG: deps[$file]: ${deps[$i]}"
done
function depends {
rhs="$(package ${generated_files[$2]})"
####echo "DBG: does generated_files[$1] depend on ${rhs}?"
for dep in ${deps[$1]}; do
###echo "DBG: checking against $dep"
if [[ "${dep}" == "${rhs}" ]]; then
###echo "DBG: = yes"
return 0
fi
done
###echo "DBG: = no"
return 1
}
function tsort {
visited[$1]=true
local j=0
for (( j=0; j<number; j++ )); do
if ! ${visited[${j}]}; then
if depends "$1" ${j}; then
tsort $j
fi
fi
done
result="${result} $1"
}
echo "Building dependencies"
for (( i=0; i<number; i++ )); do
###echo "DBG: considering ${generated_files[${i}]}"
if ! ${visited[${i}]}; then
###echo "DBG: tsorting ${generated_files[${i}]}"
tsort ${i}
fi
done
index=(${result})
haveindex=${index:-}
if [[ -z ${haveindex} ]]; then
echo No files found for $0
echo A previous run of $0 may have deleted all the files and then crashed.
echo Use 'touch' to create files named 'types.generated.go' listed as deleted in 'git status'
exit 1
fi
echo "Building codecgen"
make generated_files
CODECGEN="${PWD}/codecgen_binary"
go build -o "${CODECGEN}" ./vendor/github.com/ugorji/go/codec/codecgen
# Running codecgen fails if some of the files doesn't compile.
# Thus (since all the files are completely auto-generated and
# not required for the code to be compilable, we first remove
# them and the regenerate them.
for (( i=0; i < number; i++ )); do
rm -f "${generated_files[${i}]}"
done
# Generate files in the dependency order.
for current in "${index[@]}"; do
generated_file=${generated_files[${current}]}
initial_dir=${PWD}
file=${generated_file/\.generated\.go/.go}
echo "processing ${file}"
# codecgen work only if invoked from directory where the file
# is located.
pushd "$(dirname ${file})" > /dev/null
base_file=$(basename "${file}")
base_generated_file=$(basename "${generated_file}")
# We use '-d 1234' flag to have a deterministic output every time.
# The constant was just randomly chosen.
###echo "DBG: running ${CODECGEN} -d 1234 -o ${base_generated_file} ${base_file}"
${CODECGEN} -d 1234 -o "${base_generated_file}" "${base_file}"
# Add boilerplate at the beginning of the generated file.
sed 's/YEAR/2016/' "${initial_dir}/hack/boilerplate/boilerplate.go.txt" > "${base_generated_file}.tmp"
cat "${base_generated_file}" >> "${base_generated_file}.tmp"
mv "${base_generated_file}.tmp" "${base_generated_file}"
popd > /dev/null
done

67
hack/verify-codecgen.sh
View File

@@ -1,67 +0,0 @@
#!/bin/bash
# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
kube::golang::setup_env
generated_files=($(
find . -not \( \
\( \
-wholename './output' \
-o -wholename './_output' \
-o -wholename './release' \
-o -wholename './target' \
-o -wholename '*/third_party/*' \
-o -wholename '*/vendor/*' \
\) -prune \
\) -name '*.generated.go'))
function cleanup {
for generated_file in ${generated_files[@]}; do
rm -f "${generated_file}.original"
done
}
trap cleanup EXIT SIGINT
for generated_file in ${generated_files[@]}; do
cat "${generated_file}" > "${generated_file}.original"
done
${KUBE_ROOT}/hack/update-codecgen.sh
ret=0
# Generate files in the dependency order.
for generated_file in ${generated_files[@]}; do
cur=0
diff -Naupr -I 'Auto generated by' "${generated_file}" "${generated_file}.original" || cur=$?
if [[ $cur -eq 0 ]]; then
echo "${generated_file} up to date."
else
echo "${generated_file} was out of date. Please run hack/update-codecgen.sh. (If you're running locally, this was run for you already.)"
ret=1
fi
done
if [ "${ret}" -ne "0" ]; then
exit $ret
fi

7
pkg/api/serialization_test.go
View File

@@ -27,7 +27,7 @@ import (
"testing"
"github.com/golang/protobuf/proto"
"github.com/ugorji/go/codec"
jsoniter "github.com/json-iterator/go"
"k8s.io/api/core/v1"
"k8s.io/api/extensions/v1beta1"
@@ -536,7 +536,7 @@ func BenchmarkDecodeIntoJSON(b *testing.B) {
b.StopTimer()
}
// BenchmarkDecodeJSON provides a baseline for codecgen JSON decode performance
// BenchmarkDecodeJSON provides a baseline for JSON decode performance
func BenchmarkDecodeIntoJSONCodecGen(b *testing.B) {
kcodec := testapi.Default.Codec()
items := benchmarkItems(b)
@@ -549,12 +549,11 @@ func BenchmarkDecodeIntoJSONCodecGen(b *testing.B) {
}
encoded[i] = data
}
handler := &codec.JsonHandle{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
obj := v1.Pod{}
if err := codec.NewDecoderBytes(encoded[i%width], handler).Decode(&obj); err != nil {
if err := jsoniter.ConfigFastest.Unmarshal(encoded[i%width], &obj); err != nil {
b.Fatal(err)
}
}

14
staging/src/k8s.io/apimachinery/pkg/apis/meta/fuzzer/fuzzer.go
View File

@@ -145,15 +145,11 @@ func v1alpha1FuzzerFuncs(codecs runtimeserializer.CodecFactory) []interface{} {
case 2:
r.Cells[i] = c.RandBool()
case 3:
// maps roundtrip as map[interface{}]interface{}, but the json codec cannot encode that
// TODO: get maps to roundtrip properly
/*
x := map[string]interface{}{}
for j := c.Intn(10) + 1; j >= 0; j-- {
x[c.RandString()] = c.RandString()
}
r.Cells[i] = x
*/
x := map[string]interface{}{}
for j := c.Intn(10) + 1; j >= 0; j-- {
x[c.RandString()] = c.RandString()
}
r.Cells[i] = x
case 4:
x := make([]interface{}, c.Intn(10))
for i := range x {

10
staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/group_version_test.go
View File

@@ -21,7 +21,7 @@ import (
"reflect"
"testing"
"github.com/ugorji/go/codec"
jsoniter "github.com/json-iterator/go"
)
type GroupVersionHolder struct {
@@ -46,12 +46,12 @@ func TestGroupVersionUnmarshalJSON(t *testing.T) {
if !reflect.DeepEqual(result.GV, c.expect) {
t.Errorf("JSON codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
}
// test the Ugorji codec
if err := codec.NewDecoderBytes(c.input, new(codec.JsonHandle)).Decode(&result); err != nil {
t.Errorf("Ugorji codec failed to unmarshal input '%v': %v", c.input, err)
// test the json-iterator codec
if err := jsoniter.ConfigFastest.Unmarshal(c.input, &result); err != nil {
t.Errorf("json-iterator codec failed to unmarshal input '%v': %v", c.input, err)
}
if !reflect.DeepEqual(result.GV, c.expect) {
t.Errorf("Ugorji codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
t.Errorf("json-iterator codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
}
}
}

4
staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_test.go
View File

@@ -21,7 +21,7 @@ import (
"reflect"
"testing"
"github.com/ugorji/go/codec"
jsoniter "github.com/json-iterator/go"
)
func TestVerbsUgorjiMarshalJSON(t *testing.T) {
@@ -58,7 +58,7 @@ func TestVerbsUgorjiUnmarshalJSON(t *testing.T) {
for i, c := range cases {
var result APIResource
if err := codec.NewDecoderBytes([]byte(c.input), new(codec.JsonHandle)).Decode(&result); err != nil {
if err := jsoniter.ConfigFastest.Unmarshal([]byte(c.input), &result); err != nil {
t.Errorf("[%d] Failed to unmarshal input '%v': %v", i, c.input, err)
}
if !reflect.DeepEqual(result, c.result) {

42
staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go
View File

@@ -19,9 +19,11 @@ package json
import (
"encoding/json"
"io"
"strconv"
"unsafe"
"github.com/ghodss/yaml"
"github.com/ugorji/go/codec"
jsoniter "github.com/json-iterator/go"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -66,6 +68,36 @@ type Serializer struct {
var _ runtime.Serializer = &Serializer{}
var _ recognizer.RecognizingDecoder = &Serializer{}
func init() {
// Force jsoniter to decode number to interface{} via ints, if possible.
decodeNumberAsInt64IfPossible := func(ptr unsafe.Pointer, iter *jsoniter.Iterator) {
switch iter.WhatIsNext() {
case jsoniter.NumberValue:
var number json.Number
iter.ReadVal(&number)
u64, err := strconv.ParseUint(string(number), 10, 64)
if err == nil {
*(*interface{})(ptr) = u64
return
}
i64, err := strconv.ParseInt(string(number), 10, 64)
if err == nil {
*(*interface{})(ptr) = i64
return
}
f64, err := strconv.ParseFloat(string(number), 64)
if err == nil {
*(*interface{})(ptr) = f64
return
}
// Not much we can do here.
default:
*(*interface{})(ptr) = iter.Read()
}
}
jsoniter.RegisterTypeDecoderFunc("interface {}", decodeNumberAsInt64IfPossible)
}
// Decode attempts to convert the provided data into YAML or JSON, extract the stored schema kind, apply the provided default gvk, and then
// load that data into an object matching the desired schema kind or the provided into. If into is *runtime.Unknown, the raw data will be
// extracted and no decoding will be performed. If into is not registered with the typer, then the object will be straight decoded using
@@ -121,7 +153,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
types, _, err := s.typer.ObjectKinds(into)
switch {
case runtime.IsNotRegisteredError(err):
if err := codec.NewDecoderBytes(data, new(codec.JsonHandle)).Decode(into); err != nil {
if err := jsoniter.ConfigFastest.Unmarshal(data, into); err != nil {
return nil, actual, err
}
return into, actual, nil
@@ -155,7 +187,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
return nil, actual, err
}
if err := codec.NewDecoderBytes(data, new(codec.JsonHandle)).Decode(obj); err != nil {
if err := jsoniter.ConfigFastest.Unmarshal(data, obj); err != nil {
return nil, actual, err
}
return obj, actual, nil
@@ -164,7 +196,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
// Encode serializes the provided object to the given writer.
func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error {
if s.yaml {
json, err := json.Marshal(obj)
json, err := jsoniter.ConfigFastest.Marshal(obj)
if err != nil {
return err
}
@@ -177,7 +209,7 @@ func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error {
}
if s.pretty {
data, err := json.MarshalIndent(obj, "", " ")
data, err := jsoniter.ConfigFastest.MarshalIndent(obj, "", " ")
if err != nil {
return err
}