From 25fca3d68d16a875c708fc71b5049df20f4a208a Mon Sep 17 00:00:00 2001 From: Andrew Rynhard Date: Tue, 15 Jan 2019 18:46:41 -0800 Subject: [PATCH] feat: import core service containers from local store (#309) Signed-off-by: Andrew Rynhard --- .dockerignore | 1 + .gitignore | 1 + Dockerfile | 2 + Makefile | 26 +++- go.mod | 6 +- go.sum | 12 -- hack/scripts/push.sh | 18 --- internal/app/init/main.go | 112 +++++++++++++++--- .../system/runner/containerd/containerd.go | 10 +- internal/app/init/pkg/system/runner/runner.go | 14 ++- .../app/init/pkg/system/services/blockd.go | 8 +- .../app/init/pkg/system/services/kubeadm.go | 10 +- .../app/init/pkg/system/services/kubelet.go | 10 +- internal/app/init/pkg/system/services/osd.go | 11 +- .../app/init/pkg/system/services/proxyd.go | 11 +- .../app/init/pkg/system/services/trustd.go | 8 +- internal/app/osctl/cmd/logs.go | 14 ++- internal/app/osctl/cmd/ps.go | 10 +- internal/app/osctl/cmd/restart.go | 13 +- internal/app/osctl/cmd/root.go | 2 +- internal/app/osctl/internal/client/client.go | 4 +- internal/app/osd/internal/reg/reg.go | 27 +++-- internal/app/osd/proto/api.proto | 14 ++- internal/pkg/constants/constants.go | 5 +- internal/pkg/userdata/userdata.go | 3 +- 25 files changed, 219 insertions(+), 133 deletions(-) delete mode 100755 hack/scripts/push.sh diff --git a/.dockerignore b/.dockerignore index ca8923d31..2e23fbda4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ ** !hack +!images !installer !internal !vendor diff --git a/.gitignore b/.gitignore index 7d39e636b..259e388df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ build cache +images vendor .vscode diff --git a/Dockerfile b/Dockerfile index a76a3078b..ec6d4ae80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -98,6 +98,8 @@ RUN curl -L https://github.com/containernetworking/plugins/releases/download/v0. # kubeadm RUN curl --retry 3 --retry-delay 60 -L https://storage.googleapis.com/kubernetes-release/release/v1.13.1/bin/linux/amd64/kubeadm -o /rootfs/bin/kubeadm RUN chmod +x /rootfs/bin/kubeadm +# images +COPY images /rootfs/usr/images # udevd COPY --from=udevd-build /udevd /rootfs/bin/udevd # cleanup diff --git a/Makefile b/Makefile index 875c9656c..2926ad87f 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ COMMON_APP_ARGS := -f ./Dockerfile --build-arg TOOLCHAIN_VERSION=690a03a --build export DOCKER_BUILDKIT := 1 -all: enforce rootfs initramfs osd osctl trustd proxyd blockd udevd test installer docs +all: enforce rootfs initramfs osctl udevd test installer docs enforce: @docker run --rm -it -v $(PWD):/src -w /src autonomy/conform:latest @@ -16,6 +16,7 @@ osd: -t autonomy/$@:$(TAG) \ --target=$@ \ $(COMMON_APP_ARGS) + @docker save autonomy/$@:$(TAG) -o ./images/$@.tar osctl: @docker build \ @@ -30,18 +31,21 @@ trustd: -t autonomy/$@:$(TAG) \ --target=$@ \ $(COMMON_APP_ARGS) + @docker save autonomy/$@:$(TAG) -o ./images/$@.tar proxyd: @docker build \ -t autonomy/$@:$(TAG) \ --target=$@ \ $(COMMON_APP_ARGS) + @docker save autonomy/$@:$(TAG) -o ./images/$@.tar blockd: @docker build \ -t autonomy/$@:$(TAG) \ --target=$@ \ $(COMMON_APP_ARGS) + @docker save autonomy/$@:$(TAG) -o ./images/$@.tar udevd: @docker build \ @@ -55,7 +59,23 @@ test: --target=$@ \ $(COMMON_APP_ARGS) -rootfs: +hyperkube: + @docker pull k8s.gcr.io/$@:v1.13.1 + @docker save k8s.gcr.io/$@:v1.13.1 -o ./images/$@.tar + +etcd: + @docker pull k8s.gcr.io/$@:3.2.24 + @docker save k8s.gcr.io/$@:3.2.24 -o ./images/$@.tar + +coredns: + @docker pull k8s.gcr.io/$@:1.2.6 + @docker save k8s.gcr.io/$@:1.2.6 -o ./images/$@.tar + +pause: + @docker pull k8s.gcr.io/$@:3.1 + @docker save k8s.gcr.io/$@:3.1 -o ./images/$@.tar + +rootfs: hyperkube etcd coredns pause osd trustd proxyd blockd @docker build \ -t autonomy/$@:$(TAG) \ --target=$@ \ @@ -79,7 +99,7 @@ docs: @docker run --rm -it -v $(PWD):/out autonomy/$@:$(TAG) cp -R /docs /out .PHONY: installer -installer: +installer: initramfs rootfs @docker build \ -t autonomy/talos:$(TAG) \ --target=$@ \ diff --git a/go.mod b/go.mod index e514658e1..039579e73 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,12 @@ module github.com/autonomy/talos require ( - github.com/BurntSushi/toml v0.3.0 // indirect github.com/Microsoft/go-winio v0.4.9 // indirect github.com/Microsoft/hcsshim v0.7.0 // indirect github.com/containerd/cgroups v0.0.0-20180905221500-58556f5ad844 github.com/containerd/containerd v1.2.1 github.com/containerd/continuity v0.0.0-20181003075958-be9bd761db19 // indirect - github.com/containerd/cri v1.11.1 // indirect + github.com/containerd/cri v1.11.1 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260 // indirect github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd github.com/coreos/go-systemd v0.0.0-20180828140353-eee3db372b31 // indirect @@ -41,7 +40,6 @@ require ( github.com/onsi/gomega v1.4.1 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect - github.com/opencontainers/runc v0.1.1 // indirect github.com/opencontainers/runtime-spec v0.1.2-0.20180710222632-d810dbc60d8c github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.8.0 @@ -55,7 +53,7 @@ require ( github.com/syndtr/gocapability v0.0.0-20180223013746-33e07d32887e // indirect github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728 golang.org/x/crypto v0.0.0-20180515001509-1a580b3eff78 // indirect - golang.org/x/net v0.0.0-20181201002055-351d144fa1fc + golang.org/x/net v0.0.0-20181201002055-351d144fa1fc // indirect golang.org/x/oauth2 v0.0.0-20181003184128-c57b0facaced // indirect golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect golang.org/x/sys v0.0.0-20181019160139-8e24a49d80f8 diff --git a/go.sum b/go.sum index cb988a428..c687dc077 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.0 h1:e1/Ivsx3Z0FVTV0NSOv/aVgbUWyQuzj7DDnFblkRvsY= -github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.4.9 h1:3RbgqgGVqmcpbOiwrjbVtDHLlJBGF6aE+yHmNtBNsFQ= github.com/Microsoft/go-winio v0.4.9/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/hcsshim v0.7.0 h1:m1J6JDH52fG9Qjq8fznVe8PNX75RFge88bzQ8u/HFM0= @@ -10,7 +8,6 @@ github.com/containerd/cgroups v0.0.0-20180905221500-58556f5ad844 h1:W0F6ErEE8B84 github.com/containerd/cgroups v0.0.0-20180905221500-58556f5ad844/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= github.com/containerd/containerd v1.2.1 h1:rG4/dK9V2qa5a9ly/E3CtG6/FBXfmSkDo8An3ea2Yt8= github.com/containerd/containerd v1.2.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.2.2 h1:N3tAHxrX+byqfAsENdDWLSMtFD4thUxK7kFElUl+8z8= github.com/containerd/continuity v0.0.0-20181003075958-be9bd761db19 h1:HSgjWPBWohO3kHDPwCPUGSLqJjXCjA7ad5057beR2ZU= github.com/containerd/continuity v0.0.0-20181003075958-be9bd761db19/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/cri v1.11.1 h1:mR8+eNW4zEcbWGTGEpmDd7GzMmK7IMxMSVAZ2aIDKA4= @@ -86,8 +83,6 @@ github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2i github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runtime-spec v0.1.2-0.20180710222632-d810dbc60d8c h1:Sl3OOVnd2RrFa6FZTRgEFtvKRvc6VaLDpbZu4vlvSiY= github.com/opencontainers/runtime-spec v0.1.2-0.20180710222632-d810dbc60d8c/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= @@ -139,8 +134,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180831171423-11092d34479b h1:lohp5blsw53GBXtLyLNaTXPXS9pJ1tiTw61ZHUoE9Qw= -google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181221175505-bd9b4fb69e2f h1:eT3B0O2ghdSPzjAOznr3oOLyN1HFeYUncYl7FRwg4VI= google.golang.org/genproto v0.0.0-20181221175505-bd9b4fb69e2f/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= @@ -163,15 +156,10 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.1.0+incompatible h1:5USw7CrJBYKqjg9R7QlA6jzqZKEAtvW82aNmsxxGPxw= gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.0.0-20181130031204-d04500c8c3dd h1:5aHsneN62ehs/tdtS9tWZlhVk68V7yms/Qw7nsGmvCA= -k8s.io/api v0.0.0-20181213150558-05914d821849 h1:WZFcFPXmLR7g5CxQNmjWv0mg8qulJLxDghbzS4pQtzY= -k8s.io/api v0.0.0-20181213150558-05914d821849/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20181221193117-173ce66c1e39 h1:iGq7zEPXFb0IeXAQK5RiYT1SVKX/af9F9Wv0M+yudPY= k8s.io/api v0.0.0-20181221193117-173ce66c1e39/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/apiextensions-apiserver v0.0.0-20181213153335-0fe22c71c476 h1:Ws9zfxsgV19Durts9ftyTG7TO0A/QLhmu98VqNWLiH8= k8s.io/apiextensions-apiserver v0.0.0-20181213153335-0fe22c71c476/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= -k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93 h1:tT6oQBi0qwLbbZSfDkdIsb23EwaLY85hoAV4SpXfdao= -k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.0.0-20181220065808-98853ca904e8 h1:WLypux0abPAfOJJKJNA1+g5yphAOk+ESOeSqWMwMnqA= k8s.io/apimachinery v0.0.0-20181220065808-98853ca904e8/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apiserver v0.0.0-20181213151703-3ccfe8365421 h1:NyOpnIh+7SLvC05NGCIXF9c4KhnkTZQE2SxF+m9otww= diff --git a/hack/scripts/push.sh b/hack/scripts/push.sh deleted file mode 100755 index ef5105277..000000000 --- a/hack/scripts/push.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e - -DEFAULT_REPO="autonomy" - -REPO="${1}" -TAG="${2}" - - -images=( trustd proxyd blockd osd talos ) -for i in ${images[@]}; do - if [ "${REPO}" != "${DEFAULT_REPO}" ]; then - docker tag ${DEFAULT_REPO}/${i}:${TAG} ${REPO}/${i}:${TAG} - fi - docker push ${REPO}/${i}:${TAG} -done - diff --git a/internal/app/init/main.go b/internal/app/init/main.go index dffb8d9b7..f9b77c27a 100644 --- a/internal/app/init/main.go +++ b/internal/app/init/main.go @@ -5,19 +5,25 @@ package main import ( + "context" "flag" "fmt" "log" "os" + "sync" "time" "github.com/autonomy/talos/internal/app/init/internal/platform" "github.com/autonomy/talos/internal/app/init/internal/rootfs" "github.com/autonomy/talos/internal/app/init/internal/rootfs/mount" "github.com/autonomy/talos/internal/app/init/pkg/system" + "github.com/autonomy/talos/internal/app/init/pkg/system/conditions" "github.com/autonomy/talos/internal/app/init/pkg/system/services" "github.com/autonomy/talos/internal/pkg/constants" "github.com/autonomy/talos/internal/pkg/userdata" + "github.com/containerd/containerd" + "github.com/containerd/containerd/namespaces" + criconstants "github.com/containerd/cri/pkg/constants" "github.com/pkg/errors" "golang.org/x/sys/unix" @@ -100,54 +106,130 @@ func initram() (err error) { return nil } -func root() error { +func root() (err error) { // Setup logging to /dev/kmsg. - if _, err := kmsg("[talos]"); err != nil { + if _, err = kmsg("[talos]"); err != nil { return fmt.Errorf("failed to setup logging to /dev/kmsg: %v", err) } // Read the user data. log.Printf("reading the user data: %s\n", constants.UserDataPath) - data, err := userdata.Open(constants.UserDataPath) - if err != nil { + var data *userdata.UserData + if data, err = userdata.Open(constants.UserDataPath); err != nil { return err } // Write any user specified files to disk. log.Println("writing the files specified in the user data to disk") - if err := data.WriteFiles(); err != nil { + if err = data.WriteFiles(); err != nil { return err } // Set the requested environment variables. log.Println("setting environment variables") for key, val := range data.Env { - if err := os.Setenv(key, val); err != nil { + if err = os.Setenv(key, val); err != nil { log.Printf("WARNING failed to set enivronment variable: %v", err) } } // Get a handle to the system services API. - systemservices := system.Services(data) + svcs := system.Services(data) + // Start containerd. + svcs.Start(&services.Containerd{}) + + go startSystemServices(data) + go startKubernetesServices(data) + + return nil +} + +func startSystemServices(data *userdata.UserData) { + svcs := system.Services(data) + + // Import the system images. + if err := importImages([]string{"/usr/images/blockd.tar", "/usr/images/osd.tar", "/usr/images/proxyd.tar", "/usr/images/trustd.tar"}, constants.SystemContainerdNamespace); err != nil { + panic(err) + } + + log.Println("starting system services") // Start the services common to all nodes. - log.Println("starting node services") - systemservices.Start( - &services.Containerd{}, + svcs.Start( &services.Udevd{}, &services.OSD{}, &services.Blockd{}, - &services.Kubelet{}, - &services.Kubeadm{}, ) - // Start the services common to all master nodes. if data.IsMaster() { - log.Println("starting master services") - systemservices.Start( + svcs.Start( &services.Trustd{}, &services.Proxyd{}, ) } +} + +func startKubernetesServices(data *userdata.UserData) { + svcs := system.Services(data) + + // Import the Kubernetes images. + if err := importImages([]string{"/usr/images/hyperkube.tar", "/usr/images/etcd.tar", "/usr/images/coredns.tar", "/usr/images/pause.tar"}, criconstants.K8sContainerdNamespace); err != nil { + panic(err) + } + + log.Println("starting kubernetes services") + svcs.Start( + &services.Kubelet{}, + &services.Kubeadm{}, + ) +} + +func importImages(files []string, namespace string) (err error) { + _, err = conditions.WaitForFileToExist(constants.ContainerdSocket)() + if err != nil { + return err + } + + ctx := namespaces.WithNamespace(context.Background(), namespace) + client, err := containerd.New(constants.ContainerdSocket) + if err != nil { + return err + } + // nolint: errcheck + defer client.Close() + + var wg sync.WaitGroup + + wg.Add(len(files)) + + for _, file := range files { + go func(wg *sync.WaitGroup, f string) { + defer wg.Done() + + tarball, err := os.Open(f) + if err != nil { + panic(err) + } + + imgs, err := client.Import(ctx, tarball) + if err != nil { + panic(err) + } + if err = tarball.Close(); err != nil { + panic(err) + } + + for _, img := range imgs { + image := containerd.NewImage(client, img) + log.Printf("unpacking %s (%s)\n", img.Name, img.Target.Digest) + err = image.Unpack(ctx, containerd.DefaultSnapshotter) + if err != nil { + panic(err) + } + } + }(&wg, file) + } + + wg.Wait() return nil } diff --git a/internal/app/init/pkg/system/runner/containerd/containerd.go b/internal/app/init/pkg/system/runner/containerd/containerd.go index 901411ac7..606f1c7f2 100644 --- a/internal/app/init/pkg/system/runner/containerd/containerd.go +++ b/internal/app/init/pkg/system/runner/containerd/containerd.go @@ -46,7 +46,7 @@ func WithRootfsPropagation(rp string) oci.SpecOpts { // Run implements the Runner interface. // nolint: gocyclo func (c *Containerd) Run(data *userdata.UserData, args runner.Args, setters ...runner.Option) error { - // Wait for the containerd socket. + // Wait for the containerd socket. _, err := conditions.WaitForFileToExist(constants.ContainerdSocket)() if err != nil { @@ -62,7 +62,7 @@ func (c *Containerd) Run(data *userdata.UserData, args runner.Args, setters ...r // Create the containerd client. - ctx := namespaces.WithNamespace(context.Background(), "system") + ctx := namespaces.WithNamespace(context.Background(), opts.Namespace) client, err := containerd.New(constants.ContainerdSocket) if err != nil { return err @@ -70,11 +70,9 @@ func (c *Containerd) Run(data *userdata.UserData, args runner.Args, setters ...r // nolint: errcheck defer client.Close() - // Pull the image and unpack it. - - image, err := client.Pull(ctx, opts.ContainerImage, containerd.WithPullUnpack) + image, err := client.GetImage(ctx, opts.ContainerImage) if err != nil { - return fmt.Errorf("failed to pull image %q: %v", opts.ContainerImage, err) + return err } // Create the container. diff --git a/internal/app/init/pkg/system/runner/runner.go b/internal/app/init/pkg/system/runner/runner.go index 39cb15917..41b2efa72 100644 --- a/internal/app/init/pkg/system/runner/runner.go +++ b/internal/app/init/pkg/system/runner/runner.go @@ -32,6 +32,8 @@ type Options struct { OCISpecOpts []oci.SpecOpts // ContainerImage is the container's image. ContainerImage string + // Namespace is the containerd namespace. + Namespace string // Type describes the service's restart policy. Type Type } @@ -52,8 +54,9 @@ const ( // DefaultOptions describes the default options to a runner. func DefaultOptions() *Options { return &Options{ - Env: []string{}, - Type: Forever, + Env: []string{}, + Type: Forever, + Namespace: "system", } } @@ -71,6 +74,13 @@ func WithEnv(o []string) Option { } } +// WithNamespace sets the tar file to load. +func WithNamespace(o string) Option { + return func(args *Options) { + args.Namespace = o + } +} + // WithContainerImage sets the image ref. func WithContainerImage(o string) Option { return func(args *Options) { diff --git a/internal/app/init/pkg/system/services/blockd.go b/internal/app/init/pkg/system/services/blockd.go index 167a5db9c..bac325f0d 100644 --- a/internal/app/init/pkg/system/services/blockd.go +++ b/internal/app/init/pkg/system/services/blockd.go @@ -44,13 +44,7 @@ func (t *Blockd) ConditionFunc(data *userdata.UserData) conditions.ConditionFunc } func (t *Blockd) Start(data *userdata.UserData) error { - // Set the image. - var image string - if data.Services.Blockd != nil && data.Services.Blockd.Image != "" { - image = data.Services.Blockd.Image - } else { - image = "docker.io/autonomy/blockd:" + version.Tag - } + image := "docker.io/autonomy/blockd:" + version.Tag // Set the process arguments. args := runner.Args{ diff --git a/internal/app/init/pkg/system/services/kubeadm.go b/internal/app/init/pkg/system/services/kubeadm.go index 2f8f29d3c..01a6a45af 100644 --- a/internal/app/init/pkg/system/services/kubeadm.go +++ b/internal/app/init/pkg/system/services/kubeadm.go @@ -24,6 +24,7 @@ import ( "github.com/autonomy/talos/internal/pkg/grpc/middleware/auth/basic" "github.com/autonomy/talos/internal/pkg/userdata" "github.com/containerd/containerd/oci" + criconstants "github.com/containerd/cri/pkg/constants" specs "github.com/opencontainers/runtime-spec/specs-go" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -113,13 +114,7 @@ func (k *Kubeadm) ConditionFunc(data *userdata.UserData) conditions.ConditionFun // Start implements the Service interface. // nolint: dupl func (k *Kubeadm) Start(data *userdata.UserData) error { - // Set the image. - var image string - if data.Services.Kubeadm != nil && data.Services.Kubeadm.Image != "" { - image = data.Services.Kubeadm.Image - } else { - image = constants.KubernetesImage - } + image := constants.KubernetesImage // We only wan't to run kubeadm if it hasn't been ran already. if _, err := os.Stat("/etc/kubernetes/kubelet.conf"); !os.IsNotExist(err) { @@ -159,6 +154,7 @@ func (k *Kubeadm) Start(data *userdata.UserData) error { return r.Run( data, args, + runner.WithNamespace(criconstants.K8sContainerdNamespace), runner.WithContainerImage(image), runner.WithEnv(env), runner.WithOCISpecOpts( diff --git a/internal/app/init/pkg/system/services/kubelet.go b/internal/app/init/pkg/system/services/kubelet.go index e279cfafd..07698dfa0 100644 --- a/internal/app/init/pkg/system/services/kubelet.go +++ b/internal/app/init/pkg/system/services/kubelet.go @@ -17,6 +17,7 @@ import ( "github.com/autonomy/talos/internal/pkg/constants" "github.com/autonomy/talos/internal/pkg/userdata" "github.com/containerd/containerd/oci" + criconstants "github.com/containerd/cri/pkg/constants" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -55,13 +56,7 @@ func (k *Kubelet) ConditionFunc(data *userdata.UserData) conditions.ConditionFun // Start implements the Service interface. func (k *Kubelet) Start(data *userdata.UserData) error { - // Set the image. - var image string - if data.Services.Kubelet != nil && data.Services.Kubelet.Image != "" { - image = data.Services.Kubelet.Image - } else { - image = constants.KubernetesImage - } + image := constants.KubernetesImage // Set the process arguments. args := runner.Args{ @@ -114,6 +109,7 @@ func (k *Kubelet) Start(data *userdata.UserData) error { return r.Run( data, args, + runner.WithNamespace(criconstants.K8sContainerdNamespace), runner.WithContainerImage(image), runner.WithEnv(env), runner.WithOCISpecOpts( diff --git a/internal/app/init/pkg/system/services/osd.go b/internal/app/init/pkg/system/services/osd.go index eb19e2f1c..5fab239a0 100644 --- a/internal/app/init/pkg/system/services/osd.go +++ b/internal/app/init/pkg/system/services/osd.go @@ -7,6 +7,7 @@ package services import ( "fmt" + "os" "github.com/autonomy/talos/internal/app/init/pkg/system/conditions" "github.com/autonomy/talos/internal/app/init/pkg/system/runner" @@ -29,7 +30,7 @@ func (o *OSD) ID(data *userdata.UserData) string { // PreFunc implements the Service interface. func (o *OSD) PreFunc(data *userdata.UserData) error { - return nil + return os.MkdirAll("/etc/kubernetes", os.ModeDir) } // PostFunc implements the Service interface. @@ -43,13 +44,7 @@ func (o *OSD) ConditionFunc(data *userdata.UserData) conditions.ConditionFunc { } func (o *OSD) Start(data *userdata.UserData) error { - // Set the image. - var image string - if data.Services.OSD != nil && data.Services.OSD.Image != "" { - image = data.Services.OSD.Image - } else { - image = "docker.io/autonomy/osd:" + version.Tag - } + image := "docker.io/autonomy/osd:" + version.Tag // Set the process arguments. args := runner.Args{ diff --git a/internal/app/init/pkg/system/services/proxyd.go b/internal/app/init/pkg/system/services/proxyd.go index 217e3380f..dfcab3064 100644 --- a/internal/app/init/pkg/system/services/proxyd.go +++ b/internal/app/init/pkg/system/services/proxyd.go @@ -7,6 +7,7 @@ package services import ( "fmt" + "os" "github.com/autonomy/talos/internal/app/init/pkg/system/conditions" "github.com/autonomy/talos/internal/app/init/pkg/system/runner" @@ -28,7 +29,7 @@ func (p *Proxyd) ID(data *userdata.UserData) string { // PreFunc implements the Service interface. func (p *Proxyd) PreFunc(data *userdata.UserData) error { - return nil + return os.MkdirAll("/etc/kubernetes", os.ModeDir) } // PostFunc implements the Service interface. @@ -42,13 +43,7 @@ func (p *Proxyd) ConditionFunc(data *userdata.UserData) conditions.ConditionFunc } func (p *Proxyd) Start(data *userdata.UserData) error { - // Set the image. - var image string - if data.Services.Proxyd != nil && data.Services.Proxyd.Image != "" { - image = data.Services.Proxyd.Image - } else { - image = "docker.io/autonomy/proxyd:" + version.Tag - } + image := "docker.io/autonomy/proxyd:" + version.Tag // Set the process arguments. args := runner.Args{ diff --git a/internal/app/init/pkg/system/services/trustd.go b/internal/app/init/pkg/system/services/trustd.go index 8c17274e9..4fc2014bc 100644 --- a/internal/app/init/pkg/system/services/trustd.go +++ b/internal/app/init/pkg/system/services/trustd.go @@ -43,13 +43,7 @@ func (t *Trustd) ConditionFunc(data *userdata.UserData) conditions.ConditionFunc } func (t *Trustd) Start(data *userdata.UserData) error { - // Set the image. - var image string - if data.Services.Trustd != nil && data.Services.Trustd.Image != "" { - image = data.Services.Trustd.Image - } else { - image = "docker.io/autonomy/trustd:" + version.Tag - } + image := "docker.io/autonomy/trustd:" + version.Tag // Set the process arguments. args := runner.Args{ diff --git a/internal/app/osctl/cmd/logs.go b/internal/app/osctl/cmd/logs.go index 8a6ebdac5..de65ec697 100644 --- a/internal/app/osctl/cmd/logs.go +++ b/internal/app/osctl/cmd/logs.go @@ -11,6 +11,7 @@ import ( "github.com/autonomy/talos/internal/app/osctl/internal/client" "github.com/autonomy/talos/internal/app/osd/proto" "github.com/autonomy/talos/internal/pkg/constants" + criconstants "github.com/containerd/cri/pkg/constants" "github.com/spf13/cobra" ) @@ -26,7 +27,6 @@ var logsCmd = &cobra.Command{ } os.Exit(1) } - process := args[0] creds, err := client.NewDefaultClientCredentials() if err != nil { fmt.Println(err) @@ -37,9 +37,15 @@ var logsCmd = &cobra.Command{ fmt.Print(err) os.Exit(1) } + var namespace string + if kubernetes { + namespace = criconstants.K8sContainerdNamespace + } else { + namespace = constants.SystemContainerdNamespace + } r := &proto.LogsRequest{ - Process: process, - Container: isContainer, + Id: args[0], + Namespace: namespace, } if err := c.Logs(r); err != nil { fmt.Print(err) @@ -49,6 +55,6 @@ var logsCmd = &cobra.Command{ } func init() { - logsCmd.Flags().BoolVarP(&isContainer, "container", "c", false, "treat as a container ID") + logsCmd.Flags().BoolVarP(&kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace") rootCmd.AddCommand(logsCmd) } diff --git a/internal/app/osctl/cmd/ps.go b/internal/app/osctl/cmd/ps.go index 73888d394..12585c5aa 100644 --- a/internal/app/osctl/cmd/ps.go +++ b/internal/app/osctl/cmd/ps.go @@ -11,6 +11,7 @@ import ( "github.com/autonomy/talos/internal/app/osctl/internal/client" "github.com/autonomy/talos/internal/pkg/constants" + criconstants "github.com/containerd/cri/pkg/constants" "github.com/spf13/cobra" ) @@ -30,7 +31,13 @@ var psCmd = &cobra.Command{ fmt.Println(err) os.Exit(1) } - if err := c.Processes(); err != nil { + var namespace string + if kubernetes { + namespace = criconstants.K8sContainerdNamespace + } else { + namespace = constants.SystemContainerdNamespace + } + if err := c.Processes(namespace); err != nil { fmt.Println(err) os.Exit(1) } @@ -38,5 +45,6 @@ var psCmd = &cobra.Command{ } func init() { + psCmd.Flags().BoolVarP(&kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace") rootCmd.AddCommand(psCmd) } diff --git a/internal/app/osctl/cmd/restart.go b/internal/app/osctl/cmd/restart.go index 9defb57d4..d09d14433 100644 --- a/internal/app/osctl/cmd/restart.go +++ b/internal/app/osctl/cmd/restart.go @@ -12,6 +12,7 @@ import ( "github.com/autonomy/talos/internal/app/osctl/internal/client" "github.com/autonomy/talos/internal/app/osd/proto" "github.com/autonomy/talos/internal/pkg/constants" + criconstants "github.com/containerd/cri/pkg/constants" "github.com/spf13/cobra" ) @@ -41,9 +42,16 @@ var restartCmd = &cobra.Command{ fmt.Println(err) os.Exit(1) } + var namespace string + if kubernetes { + namespace = criconstants.K8sContainerdNamespace + } else { + namespace = constants.SystemContainerdNamespace + } r := &proto.RestartRequest{ - Id: args[0], - Timeout: timeout, + Id: args[0], + Namespace: namespace, + Timeout: timeout, } if err := c.Restart(r); err != nil { fmt.Println(err) @@ -54,5 +62,6 @@ var restartCmd = &cobra.Command{ func init() { restartCmd.Flags().Int32VarP(&timeout, "timeout", "t", 60, "the timeout duration in seconds") + restartCmd.Flags().BoolVarP(&kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace") rootCmd.AddCommand(restartCmd) } diff --git a/internal/app/osctl/cmd/root.go b/internal/app/osctl/cmd/root.go index 4d2325418..1e269d360 100644 --- a/internal/app/osctl/cmd/root.go +++ b/internal/app/osctl/cmd/root.go @@ -15,13 +15,13 @@ var ( ca string crt string key string - isContainer bool organization string rsa bool name string csr string ip string hours int + kubernetes bool ) // rootCmd represents the base command when called without any subcommands diff --git a/internal/app/osctl/internal/client/client.go b/internal/app/osctl/internal/client/client.go index d10b73637..fb51bd769 100644 --- a/internal/app/osctl/internal/client/client.go +++ b/internal/app/osctl/internal/client/client.go @@ -117,9 +117,9 @@ func (c *Client) Kubeconfig() (err error) { } // Processes implements the proto.OSDClient interface. -func (c *Client) Processes() (err error) { +func (c *Client) Processes(namespace string) (err error) { ctx := context.Background() - reply, err := c.client.Processes(ctx, &empty.Empty{}) + reply, err := c.client.Processes(ctx, &proto.ProcessesRequest{Namespace: namespace}) if err != nil { return } diff --git a/internal/app/osd/internal/reg/reg.go b/internal/app/osd/internal/reg/reg.go index 9088e43a2..f6283b27f 100644 --- a/internal/app/osd/internal/reg/reg.go +++ b/internal/app/osd/internal/reg/reg.go @@ -12,9 +12,7 @@ import ( "github.com/autonomy/talos/internal/app/init/pkg/system/runner" containerdrunner "github.com/autonomy/talos/internal/app/init/pkg/system/runner/containerd" - servicelog "github.com/autonomy/talos/internal/app/init/pkg/system/runner/process/log" "github.com/autonomy/talos/internal/app/osd/proto" - "github.com/autonomy/talos/internal/pkg/chunker" filechunker "github.com/autonomy/talos/internal/pkg/chunker/file" "github.com/autonomy/talos/internal/pkg/constants" "github.com/autonomy/talos/internal/pkg/userdata" @@ -59,8 +57,8 @@ func (r *Registrator) Kubeconfig(ctx context.Context, in *empty.Empty) (data *pr } // Processes implements the proto.OSDServer interface. -func (r *Registrator) Processes(ctx context.Context, in *empty.Empty) (reply *proto.ProcessesReply, err error) { - ctx = namespaces.WithNamespace(ctx, "system") +func (r *Registrator) Processes(ctx context.Context, in *proto.ProcessesRequest) (reply *proto.ProcessesReply, err error) { + ctx = namespaces.WithNamespace(ctx, in.Namespace) client, err := containerd.New(constants.ContainerdSocket) if err != nil { return nil, err @@ -124,7 +122,7 @@ func (r *Registrator) Processes(ctx context.Context, in *empty.Empty) (reply *pr // Restart implements the proto.OSDServer interface. func (r *Registrator) Restart(ctx context.Context, in *proto.RestartRequest) (reply *proto.RestartReply, err error) { - ctx = namespaces.WithNamespace(ctx, "system") + ctx = namespaces.WithNamespace(ctx, in.Namespace) client, err := containerd.New(constants.ContainerdSocket) if err != nil { return nil, err @@ -226,18 +224,23 @@ func (r *Registrator) Dmesg(ctx context.Context, in *empty.Empty) (data *proto.D // Logs implements the proto.OSDServer interface. Service or container logs can // be requested and the contents of the log file are streamed in chunks. func (r *Registrator) Logs(req *proto.LogsRequest, l proto.OSD_LogsServer) (err error) { - var chunk chunker.ChunkReader - if req.Container { - return errors.New("container logs is currently unsupported") + ctx := namespaces.WithNamespace(context.Background(), req.Namespace) + client, err := containerd.New(constants.ContainerdSocket) + if err != nil { + return err } - - logpath := servicelog.FormatLogPath(req.Process) - file, _err := os.OpenFile(logpath, os.O_RDONLY, 0) + // nolint: errcheck + defer client.Close() + task, err := client.TaskService().Get(ctx, &tasks.GetRequest{ContainerID: req.Id}) + if err != nil { + return err + } + file, _err := os.OpenFile(task.Process.Stdout, os.O_RDONLY, 0) if _err != nil { err = _err return } - chunk = filechunker.NewChunker(file) + chunk := filechunker.NewChunker(file) if chunk == nil { return errors.New("no log reader found") diff --git a/internal/app/osd/proto/api.proto b/internal/app/osd/proto/api.proto index 75269e635..8726b31e1 100644 --- a/internal/app/osd/proto/api.proto +++ b/internal/app/osd/proto/api.proto @@ -8,7 +8,7 @@ import "google/protobuf/empty.proto"; // The OSD service definition. service OSD { rpc Kubeconfig(google.protobuf.Empty) returns (Data) {} - rpc Processes(google.protobuf.Empty) returns (ProcessesReply) {} + rpc Processes(ProcessesRequest) returns (ProcessesReply) {} rpc Restart(RestartRequest) returns (RestartReply) {} rpc Reset(google.protobuf.Empty) returns (ResetReply) {} rpc Reboot(google.protobuf.Empty) returns (RebootReply) {} @@ -17,6 +17,11 @@ service OSD { rpc Version(google.protobuf.Empty) returns (Data) {} } +// The request message containing the containerd namespace. +message ProcessesRequest { + string namespace = 1; +} + // The response message containing the requested processes. message ProcessesReply { repeated Process processes = 1; @@ -34,7 +39,8 @@ message Process { // The request message containing the process to restart. message RestartRequest { string id = 1; - int32 timeout = 2; + string namespace = 2; + int32 timeout = 3; } // The response message containing the restart status. @@ -48,8 +54,8 @@ message RebootReply {} // The request message containing the process name. message LogsRequest { - string process = 1; - bool container = 2; + string id = 1; + string namespace = 2; } // The response message containing the requested logs. diff --git a/internal/pkg/constants/constants.go b/internal/pkg/constants/constants.go index 4302c0166..d18654f3f 100644 --- a/internal/pkg/constants/constants.go +++ b/internal/pkg/constants/constants.go @@ -53,7 +53,7 @@ const ( KubernetesVersion = "v1.13.1" // KubernetesImage is the enforced hyperkube image to use for the control plane. - KubernetesImage = "gcr.io/google_containers/hyperkube:" + KubernetesVersion + KubernetesImage = "k8s.gcr.io/hyperkube:" + KubernetesVersion // DockerImage is the docker image to use as the container runtime for // Kubernetes. @@ -87,6 +87,9 @@ const ( // TrustdPort is the port for the trustd service. TrustdPort = 50001 + + // SystemContainerdNamespace is the Containerd namespace for Talos services. + SystemContainerdNamespace = "system" ) // See https://linux.die.net/man/3/klogctl diff --git a/internal/pkg/userdata/userdata.go b/internal/pkg/userdata/userdata.go index 8bac175fe..aba87dca0 100644 --- a/internal/pkg/userdata/userdata.go +++ b/internal/pkg/userdata/userdata.go @@ -233,8 +233,7 @@ type CRT struct { // CommonServiceOptions represents the set of options common to all services. type CommonServiceOptions struct { - Image string `yaml:"image,omitempty"` - Env Env `yaml:"env,omitempty"` + Env Env `yaml:"env,omitempty"` } // WriteFiles writes the requested files to disk.