adapter kube version 1.26

2026-01-27 10:18:13 +00:00 · 2023-10-27 10:46:24 +08:00
parent 36ffb7fc3f
commit 687e87921b
12 changed files with 663 additions and 116 deletions
--- a/README.md
+++ b/README.md
@@ -36,10 +36,59 @@ export RUNTIME=XXX
 export ALICLOUD_REGION=XXX
 export ALICLOUD_ACCESS_KEY=XXX
 export ALICLOUD_SECRET_KEY=XXX
-packer build examples/ack-optimized-os-1.18.json
+packer build examples/ack-optimized-os-all.json
 ```
 NOTE: `RUNTIME` only support `docker` and `containerd`

+```shell
+{
+  "variables": {
+    "image_name": "ack-optimized_image-1.20-{{timestamp}}",
+    "source_image": "aliyun_2_1903_x64_20G_alibase_20210120.vhd",
+    "instance_type": "ecs.gn6i-c4g1.xlarge",
+    "region": "{{env `ALICLOUD_REGION`}}",
+    "access_key": "{{env `ALICLOUD_ACCESS_KEY`}}",
+    "secret_key": "{{env `ALICLOUD_SECRET_KEY`}}",
+    "runtime": "{{env `RUNTIME`}}",
+    "skip_secrutiy_fix": "{{env `SKIP_SECURITY_FIX`}}"
+  },
+  "builders": [
+    {
+      "type": "alicloud-ecs",
+      "access_key": "{{user `access_key`}}",
+      "secret_key": "{{user `secret_key`}}",
+      "region": "{{user `region`}}",
+      "image_name": "{{user `image_name`}}",
+      "source_image": "{{user `source_image`}}",
+      "ssh_username": "root",
+      "instance_type": "{{user `instance_type`}}",
+      "skip_image_validation": "true",
+      "io_optimized": "true"
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "file",
+      "source": "scripts/ack-optimized-os-all.sh",
+      "destination": "/root/"
+    },
+    {
+      "type": "shell",
+      "inline": [
+        "export RUNTIME={{user `runtime`}}",
+        "export SKIP_SECURITY_FIX={{user `skip_secrutiy_fix`}}",
+        "export OS_ARCH=amd64",
+        "export PRESET_GPU=true",    # If you want to download gpu, set PRESET_GPU to true and also set instance_type to gpu instance, supports version 1.20+.
+        "export NVIDIA_DRIVER_VERSION=460.106.00",   #  You can set the gpu version, default is 460.91.03
+        "export KEEP_IMAGE_DATA=true",   #  If you cache images, you must set KEEP_IMAGE_DATA to true
+        "export KUBE_VERSION=1.26.3-aliyun.1",   #  Set KUBE_VERSION according to your cluster version
+        "bash /root/ack-optimized-os-all.sh",
+        "ctr -n k8s.io i pull docker.io/library/nginx:1.7.9"  #  You can cache images into OS image
+      ]
+    }
+  ]
+}
+```

 ## RAM Policy

--- a/examples/ack-optimized-os-1.18.json
+++ b/examples/ack-optimized-os-1.18.json
@@ -34,6 +34,7 @@
      "inline": [
        "export RUNTIME={{user `runtime`}}",
        "export SKIP_SECURITY_FIX={{user `skip_secrutiy_fix`}}",
+        "export OS_ARCH=amd64",
        "bash /root/ack-optimized-os-1.18.sh"
      ]
    }
--- a/examples/ack-optimized-os-1.20.json
+++ b/examples/ack-optimized-os-1.20.json
@@ -34,6 +34,7 @@
      "inline": [
        "export RUNTIME={{user `runtime`}}",
        "export SKIP_SECURITY_FIX={{user `skip_secrutiy_fix`}}",
+        "export OS_ARCH=amd64",
        "bash /root/ack-optimized-os-1.20.sh"
      ]
    }
--- a/examples/ack-optimized-os-all.json
+++ b/examples/ack-optimized-os-all.json
@@ -0,0 +1,47 @@
+{
+  "variables": {
+    "image_name": "ack-optimized_image-1.20-{{timestamp}}",
+    "source_image": "aliyun_2_1903_x64_20G_alibase_20210120.vhd",
+    "instance_type": "ecs.gn6i-c4g1.xlarge",
+    "region": "{{env `ALICLOUD_REGION`}}",
+    "access_key": "{{env `ALICLOUD_ACCESS_KEY`}}",
+    "secret_key": "{{env `ALICLOUD_SECRET_KEY`}}",
+    "runtime": "{{env `RUNTIME`}}",
+    "skip_secrutiy_fix": "{{env `SKIP_SECURITY_FIX`}}"
+  },
+  "builders": [
+    {
+      "type": "alicloud-ecs",
+      "access_key": "{{user `access_key`}}",
+      "secret_key": "{{user `secret_key`}}",
+      "region": "{{user `region`}}",
+      "image_name": "{{user `image_name`}}",
+      "source_image": "{{user `source_image`}}",
+      "ssh_username": "root",
+      "instance_type": "{{user `instance_type`}}",
+      "skip_image_validation": "true",
+      "io_optimized": "true"
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "file",
+      "source": "scripts/ack-optimized-os-all.sh",
+      "destination": "/root/"
+    },
+    {
+      "type": "shell",
+      "inline": [
+        "export RUNTIME={{user `runtime`}}",
+        "export SKIP_SECURITY_FIX={{user `skip_secrutiy_fix`}}",
+        "export OS_ARCH=amd64",
+        "export PRESET_GPU=true",
+        "export NVIDIA_DRIVER_VERSION=460.106.00",
+        "export KEEP_IMAGE_DATA=true",
+        "export KUBE_VERSION=1.26.3-aliyun.1",
+        "bash /root/ack-optimized-os-all.sh",
+        "ctr -n k8s.io i pull docker.io/library/nginx:1.7.9"
+      ]
+    }
+  ]
+}
--- a/scripts/ack-optimized-os-1.16.sh
+++ b/scripts/ack-optimized-os-1.16.sh
@@ -63,13 +63,6 @@ install_pkg() {
    public::common::install_package
 }

-preset_gpu() {
-    GPU_PACKAGE_URL=http://aliacs-k8s-${REGION}.oss-${REGION}-internal.aliyuncs.com/public/pkg
-    if [[ "$PRESET_GPU" == "true" ]]; then
-        bash -x pkg/run/$KUBE_VERSION/bin/nvidia-gpu-installer.sh --package-url-prefix ${GPU_PACKAGE_URL}
-    fi
-}
-
 trim_os() {
    local pkg_list="acl
 aic94xx-firmware
@@ -228,7 +221,7 @@ main() {
    download_pkg
    source_file
    install_pkg
-    preset_gpu
+
    pull_image
    update_os_release
    record_k8s_version
--- a/scripts/ack-optimized-os-1.18.sh
+++ b/scripts/ack-optimized-os-1.18.sh
@@ -39,8 +39,10 @@ check_params() {
 setup_env() {
    export RUNTIME
    export OS="AliyunOS"
-    export RUNTIME_VERSION="1.5.10"
-    export DOCKER_VERSION="19.03.5"
+    RUNTIME_VERSION=${RUNTIME_VERSION:-1.4.4}
+    export RUNTIME_VERSION
+    DOCKER_VERSION=${DOCKER_VERSION:-19.03.5}
+    export DOCKER_VERSION
    export KUBE_VERSION="1.18.8-aliyun.1"
    export REGION=$(curl --retry 10 -sSL http://100.100.100.200/latest/meta-data/region-id)
    export PKG_FILE_SERVER="http://aliacs-k8s-$REGION.oss-$REGION-internal.aliyuncs.com/$BETA_VERSION"
@@ -62,13 +64,6 @@ install_pkg() {
    public::common::install_package
 }

-preset_gpu() {
-    GPU_PACKAGE_URL=http://aliacs-k8s-${REGION}.oss-${REGION}-internal.aliyuncs.com/public/pkg
-    if [[ "$PRESET_GPU" == "true" ]]; then
-        bash -x pkg/run/$KUBE_VERSION/bin/nvidia-gpu-installer.sh --package-url-prefix ${GPU_PACKAGE_URL}
-    fi
-}
-
 trim_os() {
    local pkg_list="acl
 aic94xx-firmware
@@ -227,7 +222,6 @@ main() {
    download_pkg
    source_file
    install_pkg
-    preset_gpu
    pull_image
    update_os_release
    record_k8s_version
--- a/scripts/ack-optimized-os-1.20.sh
+++ b/scripts/ack-optimized-os-1.20.sh
@@ -39,17 +39,26 @@ check_params() {
 setup_env() {
    export RUNTIME
    export OS="AliyunOS"
-    export RUNTIME_VERSION="1.5.10"
-    export DOCKER_VERSION="19.03.5"
    export CLOUD_TYPE="public"
    export KUBE_VERSION="1.20.11-aliyun.1"
    export REGION=$(curl --retry 10 -sSL http://100.100.100.200/latest/meta-data/region-id)
    export PKG_FILE_SERVER="http://aliacs-k8s-$REGION.oss-$REGION-internal.aliyuncs.com/"
    export ACK_OPTIMIZED_OS_BUILD=1

-    mkdir -p /root/ack-deploy
-    cd /root/ack-deploy
+    if [[ "$RUNTIME" = "docker" ]]; then
+      RUNTIME_VERSION=${RUNTIME_VERSION:-19.03.15}
+      export RUNTIME_VERSION
+      DOCKER_VERSION=${RUNTIME_VERSION:-19.03.15}
+      export DOCKER_VERSION
+    else
+      RUNTIME_VERSION=${RUNTIME_VERSION:-1.5.13}
+      export RUNTIME_VERSION
+    fi

+    # setup k8s pull image prefix
+    if [[ -z "$KUBE_REPO_PREFIX" && -n "$REGION" ]]; then
+      export KUBE_REPO_PREFIX=registry-vpc.$REGION.aliyuncs.com/acs
+    fi
 }

 trim_os() {
@@ -162,18 +171,41 @@ xfsprogs
 }

 download_pkg() {
-    curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${KUBE_VERSION}.tar.gz -O
-    tar -zxvf run-${KUBE_VERSION}.tar.gz
+    export RELEASE_VERSION=$(echo $KUBE_VERSION | awk -F. '{print $1"."$2}')
+    curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz -O
+    tar -xvf run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz
 }

-install_pkg() {
-    ROLE=deploy-nodes pkg/run/$KUBE_VERSION/bin/kubernetes.sh
+
+source_file() {
+    ROLE=deploy-nodes pkg/run/$RELEASE_VERSION/bin/kubernetes.sh
 }

 preset_gpu() {
-    GPU_PACKAGE_URL=http://aliacs-k8s-${REGION}.oss-${REGION}-internal.aliyuncs.com/public/pkg
    if [[ "$PRESET_GPU" == "true" ]]; then
-        bash -x pkg/run/$KUBE_VERSION/bin/nvidia-gpu-installer.sh --package-url-prefix ${GPU_PACKAGE_URL}
+      for file_name in $(ls pkg/run/$RELEASE_VERSION/lib | grep -v init.sh); do
+          source pkg/run/$RELEASE_VERSION/lib/$file_name
+      done
+
+      if [[ $NVIDIA_DRIVER_VERSION == "" ]];then
+          export NVIDIA_DRIVER_VERSION=460.91.03
+      fi
+
+      nvidia::create_dir
+      # --nvidia-driver-runfile   指定驱动文件路径
+      nvidia::prepare_driver_package
+      # --nvidia-container-toolkit-rpms    指定nvidia container toolkit包含的rpm包所在目录
+      nvidia::prepare_container_runtime_package
+      # --nvidia-fabricmanager-rpm     指定nvidia fabric manager安装包（rpm格式）路径
+      nvidia::prepare_driver_package
+      # --nvidia-device-plugin-yaml     指定nvidia device plugin yaml文件路径
+      nvidia::deploy_static_pod
+
+      if [[ $RUNTIME == "docker" ]];then
+        export SKIP_CONTAINER_RUNTIME_CONFIG=true
+      fi
+
+      nvidia::gpu::installer::main
    fi
 }

@@ -182,21 +214,24 @@ pull_image() {
        systemctl start docker
        sleep 10

-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.2
-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.7.0
+        docker pull registry-${REGION}-vpc.ack.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
+        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.5
+        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.6.7
    else
        systemctl start containerd
        sleep 10

-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.2
-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.7.0
+        ctr -n k8s.io i pull registry-${REGION}-vpc.ack.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
+        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.5
+        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.6.7
    fi
 }

 update_os_release() {
-    sed -i "s#LTS#LTS ACK-Optimized-OS#" /etc/image-id
+    if [[ ! -f /etc/image-id ]]; then
+      touch /etc/image-id
+    fi
+    sed -i  "s#LTS#LTS ACK-Optimized-OS#"  /etc/image-id
 }

 record_k8s_version() {
@@ -213,6 +248,12 @@ post_install() {
    fi
 }

+keep_container_data() {
+    if [[ "$KEEP_IMAGE_DATA" = "true" ]]; then
+        touch /var/.keep-container-data
+    fi
+}
+
 cleanup() {
    rm -rf /root/ack-deploy
 }
@@ -226,9 +267,10 @@ main() {
    trim_os

    download_pkg
-    install_pkg
+    source_file
    preset_gpu
    pull_image
+    keep_container_data
    update_os_release
    record_k8s_version
    post_install
--- a/scripts/ack-optimized-os-1.22.sh
+++ b/scripts/ack-optimized-os-1.22.sh
@@ -40,20 +40,25 @@ setup_env() {
    export RUNTIME
    export OS="AliyunOS"
    export CLOUD_TYPE="public"
-    export KUBE_VERSION="1.22.10-aliyun.1"
+    export KUBE_VERSION="1.22.15-aliyun.1"
    export REGION=$(curl --retry 10 -sSL http://100.100.100.200/latest/meta-data/region-id)
    export PKG_FILE_SERVER="http://aliacs-k8s-$REGION.oss-$REGION-internal.aliyuncs.com/"
    export ACK_OPTIMIZED_OS_BUILD=1

    if [[ "$RUNTIME" = "docker" ]]; then
-      RUNTIME_VERSION="19.03.5"
-      DOCKER_VERSION="19.03.5"
+      RUNTIME_VERSION=${RUNTIME_VERSION:-19.03.15}
+      export RUNTIME_VERSION
+      DOCKER_VERSION=${RUNTIME_VERSION:-19.03.15}
+      export DOCKER_VERSION
    else
-      RUNTIME_VERSION="1.5.10"
+      RUNTIME_VERSION=${RUNTIME_VERSION:-1.5.13}
+      export RUNTIME_VERSION
    fi

-    mkdir -p /root/ack-deploy
-    cd /root/ack-deploy
+    # setup k8s pull image prefix
+    if [[ -z "$KUBE_REPO_PREFIX" && -n "$REGION" ]]; then
+      export KUBE_REPO_PREFIX=registry-vpc.$REGION.aliyuncs.com/acs
+    fi

 }

@@ -167,18 +172,41 @@ xfsprogs
 }

 download_pkg() {
-    curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${KUBE_VERSION}.tar.gz -O
-    tar -zxvf run-${KUBE_VERSION}.tar.gz
+    export RELEASE_VERSION=$(echo $KUBE_VERSION | awk -F. '{print $1"."$2}')
+    curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz -O
+    tar -xvf run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz
 }

-install_pkg() {
-    ROLE=deploy-nodes pkg/run/$KUBE_VERSION/bin/kubernetes.sh
+
+source_file() {
+    ROLE=deploy-nodes pkg/run/$RELEASE_VERSION/bin/kubernetes.sh
 }

 preset_gpu() {
-    GPU_PACKAGE_URL=http://aliacs-k8s-${REGION}.oss-${REGION}-internal.aliyuncs.com/public/pkg
    if [[ "$PRESET_GPU" == "true" ]]; then
-        bash -x pkg/run/$KUBE_VERSION/bin/nvidia-gpu-installer.sh --package-url-prefix ${GPU_PACKAGE_URL}
+      for file_name in $(ls pkg/run/$RELEASE_VERSION/lib | grep -v init.sh); do
+          source pkg/run/$RELEASE_VERSION/lib/$file_name
+      done
+
+      if [[ $NVIDIA_DRIVER_VERSION == "" ]];then
+          export NVIDIA_DRIVER_VERSION=460.91.03
+      fi
+
+      nvidia::create_dir
+      # --nvidia-driver-runfile   指定驱动文件路径
+      nvidia::prepare_driver_package
+      # --nvidia-container-toolkit-rpms    指定nvidia container toolkit包含的rpm包所在目录
+      nvidia::prepare_container_runtime_package
+      # --nvidia-fabricmanager-rpm     指定nvidia fabric manager安装包（rpm格式）路径
+      nvidia::prepare_driver_package
+      # --nvidia-device-plugin-yaml     指定nvidia device plugin yaml文件路径
+      nvidia::deploy_static_pod
+
+      if [[ $RUNTIME == "docker" ]];then
+        export SKIP_CONTAINER_RUNTIME_CONFIG=true
+      fi
+
+      nvidia::gpu::installer::main
    fi
 }

@@ -187,21 +215,24 @@ pull_image() {
        systemctl start docker
        sleep 10

-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.2
-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.7.0
+        docker pull registry-${REGION}-vpc.ack.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
+        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.5
+        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.6.7
    else
        systemctl start containerd
        sleep 10

-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.2
-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.7.0
+        ctr -n k8s.io i pull registry-${REGION}-vpc.ack.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
+        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.5
+        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.6.7
    fi
 }

 update_os_release() {
-    sed -i "s#LTS#LTS ACK-Optimized-OS#" /etc/image-id
+    if [[ ! -f /etc/image-id ]]; then
+      touch /etc/image-id
+    fi
+    sed -i  "s#LTS#LTS ACK-Optimized-OS#"  /etc/image-id
 }

 record_k8s_version() {
@@ -218,6 +249,12 @@ post_install() {
    fi
 }

+keep_container_data() {
+    if [[ "$KEEP_IMAGE_DATA" = "true" ]]; then
+        touch /var/.keep-container-data
+    fi
+}
+
 cleanup() {
    rm -rf /root/ack-deploy
 }
@@ -231,9 +268,10 @@ main() {
    trim_os

    download_pkg
-    install_pkg
+    source_file
    preset_gpu
    pull_image
+    keep_container_data
    update_os_release
    record_k8s_version
    post_install
--- a/scripts/ack-optimized-os-1.24.sh
+++ b/scripts/ack-optimized-os-1.24.sh
@@ -30,26 +30,26 @@ check_params() {
        esac
    done

-    if [[ -z $RUNTIME ]] || [[ $RUNTIME != "docker" && $RUNTIME != "containerd" ]]; then
-        echo "ERROR: RUNTIME must not be empty, only support 'docker' and 'containerd' "
+    if [[ -z $RUNTIME ]] || [[ $RUNTIME != "containerd" ]]; then
+        echo "ERROR: RUNTIME must not be empty, only support 'containerd' "
        usage
    fi
 }

 setup_env() {
-    export RUNTIME
    export OS="AliyunOS"
-    export RUNTIME_VERSION="1.5.10"
-    export DOCKER_VERSION="19.03.5"
+    RUNTIME_VERSION=${RUNTIME_VERSION:-1.6.20}
+    export RUNTIME_VERSION
    export CLOUD_TYPE="public"
-    export KUBE_VERSION="1.24.3-aliyun.1"
+    export KUBE_VERSION="1.24.6-aliyun.1"
    export REGION=$(curl --retry 10 -sSL http://100.100.100.200/latest/meta-data/region-id)
    export PKG_FILE_SERVER="http://aliacs-k8s-$REGION.oss-$REGION-internal.aliyuncs.com/"
    export ACK_OPTIMIZED_OS_BUILD=1

-    mkdir -p /root/ack-deploy
-    cd /root/ack-deploy
-
+    # setup k8s pull image prefix
+    if [[ -z "$KUBE_REPO_PREFIX" && -n "$REGION" ]]; then
+      export KUBE_REPO_PREFIX=registry-vpc.$REGION.aliyuncs.com/acs
+    fi
 }

 trim_os() {
@@ -162,41 +162,55 @@ xfsprogs
 }

 download_pkg() {
-    curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${KUBE_VERSION}.tar.gz -O
-    tar -zxvf run-${KUBE_VERSION}.tar.gz
+    export RELEASE_VERSION=$(echo $KUBE_VERSION | awk -F. '{print $1"."$2}')
+    curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz -O
+    tar -xvf run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz
 }

-install_pkg() {
-    ROLE=deploy-nodes pkg/run/$KUBE_VERSION/bin/kubernetes.sh
+
+source_file() {
+    ROLE=deploy-nodes pkg/run/$RELEASE_VERSION/bin/kubernetes.sh
 }

 preset_gpu() {
-    GPU_PACKAGE_URL=http://aliacs-k8s-${REGION}.oss-${REGION}-internal.aliyuncs.com/public/pkg
    if [[ "$PRESET_GPU" == "true" ]]; then
-        bash -x pkg/run/$KUBE_VERSION/bin/nvidia-gpu-installer.sh --package-url-prefix ${GPU_PACKAGE_URL}
+      for file_name in $(ls pkg/run/$RELEASE_VERSION/lib | grep -v init.sh); do
+          source pkg/run/$RELEASE_VERSION/lib/$file_name
+      done
+
+      if [[ $NVIDIA_DRIVER_VERSION == "" ]];then
+          export NVIDIA_DRIVER_VERSION=460.91.03
+      fi
+
+      nvidia::create_dir
+      # --nvidia-driver-runfile   指定驱动文件路径
+      nvidia::prepare_driver_package
+      # --nvidia-container-toolkit-rpms    指定nvidia container toolkit包含的rpm包所在目录
+      nvidia::prepare_container_runtime_package
+      # --nvidia-fabricmanager-rpm     指定nvidia fabric manager安装包（rpm格式）路径
+      nvidia::prepare_driver_package
+      # --nvidia-device-plugin-yaml     指定nvidia device plugin yaml文件路径
+      nvidia::deploy_static_pod
+
+      nvidia::gpu::installer::main
+
    fi
 }

 pull_image() {
-    if [[ "$RUNTIME" = "docker" ]]; then
-        systemctl start docker
-        sleep 10
+    systemctl start containerd
+    sleep 10

-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.2
-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.7.0
-    else
-        systemctl start containerd
-        sleep 10
-
-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.2
-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.7.0
-    fi
+    ctr -n k8s.io i pull registry-${REGION}-vpc.ack.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
+    ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.5
+    ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.7.0
 }

 update_os_release() {
-    sed -i "s#LTS#LTS ACK-Optimized-OS#" /etc/image-id
+    if [[ ! -f /etc/image-id ]]; then
+      touch /etc/image-id
+    fi
+    sed -i  "s#LTS#LTS ACK-Optimized-OS#"  /etc/image-id
 }

 record_k8s_version() {
@@ -213,6 +227,12 @@ post_install() {
    fi
 }

+keep_container_data() {
+    if [[ "$KEEP_IMAGE_DATA" = "true" ]]; then
+        touch /var/.keep-container-data
+    fi
+}
+
 cleanup() {
    rm -rf /root/ack-deploy
 }
@@ -226,9 +246,10 @@ main() {
    trim_os

    download_pkg
-    install_pkg
+    source_file
    preset_gpu
    pull_image
+    keep_container_data
    update_os_release
    record_k8s_version
    post_install
--- a/scripts/ack-optimized-os-1.26.sh
+++ b/scripts/ack-optimized-os-1.26.sh
@@ -0,0 +1,267 @@
+#!/bin/bash
+
+set -x
+set -e
+
+usage() {
+    cat >&2 <<-EOF
+Usage:
+    $0 -r RUNTIME [-s]
+
+Flags:
+    -r: sepcify container runtime, available value: docker and containerd
+    -s: skip security upgrade
+
+Example:
+    $0 -r docker -s
+    $0 -r docker
+    $0 -r containerd -s
+    $0 -r containerd
+EOF
+    exit 1
+}
+
+check_params() {
+    while getopts "r:sh" opt; do
+        case $opt in
+        r) RUNTIME="$OPTARG" ; ;;
+        s) SKIP_SECURITY_FIX="1" ; ;;
+        h | ?) usage ; ;;
+        esac
+    done
+
+    if [[ -z $RUNTIME ]] || [[ $RUNTIME != "containerd" ]]; then
+        echo "ERROR: RUNTIME must not be empty, only support 'containerd' "
+        usage
+    fi
+
+}
+
+setup_env() {
+    export OS="AliyunOS"
+    RUNTIME_VERSION=${RUNTIME_VERSION:-1.6.20}
+    export RUNTIME_VERSION
+    export KUBE_VERSION='1.26.3-aliyun.1'
+    export REGION=$(curl --retry 10 -sSL http://100.100.100.200/latest/meta-data/region-id)
+    export PKG_FILE_SERVER="http://aliacs-k8s-$REGION.oss-$REGION-internal.aliyuncs.com/$BETA_VERSION"
+    export ACK_OPTIMIZED_OS_BUILD=1
+
+    # setup k8s pull image prefix
+    if [[ -z "$KUBE_REPO_PREFIX" && -n "$REGION" ]]; then
+      export KUBE_REPO_PREFIX=registry-vpc.$REGION.aliyuncs.com/acs
+    fi
+}
+
+
+download_pkg() {
+    export RELEASE_VERSION=$(echo $KUBE_VERSION | awk -F. '{print $1"."$2}')
+    curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz -O
+    tar -xvf run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz
+}
+
+
+source_file() {
+    ROLE=deploy-nodes pkg/run/$RELEASE_VERSION/bin/kubernetes.sh
+}
+
+#preset_gpu() {
+#    GPU_PACKAGE_URL=http://aliacs-k8s-${REGION}.oss-${REGION}-internal.aliyuncs.com/public/pkg
+#    if [[ "$PRESET_GPU" == "true" ]]; then
+#        bash -x pkg/run/$KUBE_VERSION/bin/nvidia-gpu-installer.sh --package-url-prefix ${GPU_PACKAGE_URL}
+#    fi
+#}
+
+preset_gpu() {
+    if [[ "$PRESET_GPU" == "true" ]]; then
+      for file_name in $(ls pkg/run/$RELEASE_VERSION/lib | grep -v init.sh); do
+          source pkg/run/$RELEASE_VERSION/lib/$file_name
+      done
+
+      if [[ $NVIDIA_DRIVER_VERSION == "" ]];then
+          export NVIDIA_DRIVER_VERSION=460.91.03
+      fi
+
+      nvidia::create_dir
+      # --nvidia-driver-runfile   指定驱动文件路径
+      nvidia::prepare_driver_package
+      # --nvidia-container-toolkit-rpms    指定nvidia container toolkit包含的rpm包所在目录
+      nvidia::prepare_container_runtime_package
+      # --nvidia-fabricmanager-rpm     指定nvidia fabric manager安装包（rpm格式）路径
+      nvidia::prepare_driver_package
+      # --nvidia-device-plugin-yaml     指定nvidia device plugin yaml文件路径
+      nvidia::deploy_static_pod
+
+      nvidia::gpu::installer::main
+
+    fi
+}
+
+
+
+
+
+trim_os() {
+    local pkg_list="acl
+aic94xx-firmware
+aliyun-cli
+alsa-firmware
+alsa-lib
+alsa-tools-firmware
+authconfig
+avahi-libs
+bind-libs-lite
+bind-license
+biosdevname
+btrfs-progs
+cloud
+device-mapper-event
+device-mapper-event-libs
+dmraid
+dmraid-events
+dosfstools
+ed
+file
+firewalld
+firewalld-filesystem
+freetype
+fxload
+GeoIP
+geoipupdate
+gettext
+gettext-libs
+glibc-devel
+groff-base
+hunspell
+hunspell-en
+hunspell-en-GB
+hunspell-en-US
+ivtv-firmware
+iwl1000-firmware
+iwl100-firmware
+iwl105-firmware
+iwl135-firmware
+iwl2000-firmware
+iwl2030-firmware
+iwl3160-firmware
+iwl3945-firmware
+iwl4965-firmware
+iwl5000-firmware
+iwl5150-firmware
+iwl6000-firmware
+iwl6000g2a-firmware
+iwl6000g2b-firmware
+iwl6050-firmware
+iwl7260-firmware
+jansson
+kbd
+kbd-legacy
+kbd-misc
+libaio
+libdrm
+libmpc
+libpciaccess
+libpng
+libreport-filesystem
+lm_sensors-libs
+lsscsi
+lvm2
+m4
+mailx
+man-db
+mariadb-libs
+mdadm
+microcode_ctl
+mpfr
+NetworkManager
+NetworkManager-libnm
+NetworkManager-team
+NetworkManager-tui
+patch
+perl-Getopt-Long
+plymouth
+plymouth-scripts
+postfix
+python3
+python3-libs
+python3-pip
+python3-setuptools
+python-decorator
+python-IPy
+rng-tools
+rsync
+sgpio
+slang
+spax
+strace
+sysstat
+tcpdump
+teamd
+vim-common
+vim-enhanced
+vim-filesystem
+wl1000-firmware
+wpa_supplicant
+xfsprogs
+"
+    yum remove -y $pkg_list
+    rm -rf /lib/modules/$(uname -r)/kernel/drivers/{media,staging,gpu,usb}
+    rm -rf /boot/*-rescue-* /boot/*3.10.0* /usr/share/{doc,man} /usr/src
+}
+
+pull_image() {
+    systemctl start containerd
+    sleep 10
+
+    ctr -n k8s.io i pull registry-${REGION}-vpc.ack.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
+    ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.5
+    ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.7.0
+}
+
+update_os_release() {
+    if [[ ! -f /etc/image-id ]]; then
+      touch /etc/image-id
+    fi
+    sed -i  "s#LTS#LTS ACK-Optimized-OS#"  /etc/image-id
+}
+
+record_k8s_version() {
+    cat > /etc/ACK-Optimized-OS <<-EOF
+kubelet=$KUBE_VERSION
+docker=$DOCKER_VERSION
+EOF
+}
+
+post_install() {
+    if [[ "$SKIP_SECURITY_FIX" = "true" ]]; then
+        touch /var/.skip-security-fix
+    fi
+}
+
+keep_container_data() {
+    if [[ "$KEEP_IMAGE_DATA" = "true" ]]; then
+        touch /var/.keep-container-data
+    fi
+}
+
+cleanup() {
+    rm -rf ./{addon*,docker*,kubernetes*,pkg,run*}
+}
+
+main() {
+    trap 'cleanup' EXIT
+
+    check_params "$@"
+    setup_env
+
+    trim_os
+
+    download_pkg
+    source_file
+    preset_gpu
+    pull_image
+    keep_container_data
+    update_os_release
+    record_k8s_version
+}
+
+main "$@"
--- a/scripts/ack-optimized-os-all.sh
+++ b/scripts/ack-optimized-os-all.sh
@@ -37,25 +37,38 @@ check_params() {
 }

 setup_env() {
-    export RUNTIME
    export OS="AliyunOS"

    if [[ "$RUNTIME" = "docker" ]]; then
-      export RUNTIME_VERSION="19.03.5"
-      export DOCKER_VERSION="19.03.5"
+      RUNTIME_VERSION=${RUNTIME_VERSION:-19.03.15}
+      export RUNTIME_VERSION
+      DOCKER_VERSION=${RUNTIME_VERSION:-19.03.15}
+      export DOCKER_VERSION
    else
-      export RUNTIME_VERSION="1.5.10"
+      RUNTIME_VERSION=${RUNTIME_VERSION:-1.6.20}
+      export RUNTIME_VERSION
    fi

    export REGION=$(curl --retry 10 -sSL http://100.100.100.200/latest/meta-data/region-id)
    export PKG_FILE_SERVER="http://aliacs-k8s-$REGION.oss-$REGION-internal.aliyuncs.com/$BETA_VERSION"
    export ACK_OPTIMIZED_OS_BUILD=1
+
+    # setup k8s pull image prefix
+    if [[ -z "$KUBE_REPO_PREFIX" && -n "$REGION" ]]; then
+      export KUBE_REPO_PREFIX=registry-vpc.$REGION.aliyuncs.com/acs
+    fi
 }


 download_pkg() {
-    curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${KUBE_VERSION}.tar.gz -O
-    tar -xvf run-${KUBE_VERSION}.tar.gz
+    if [[ $(echo "${KUBE_VERSION}" | cut -d. -f1) -ge 1 && $(echo "${KUBE_VERSION}" | cut -d. -f2) -ge 20 ]]; then
+      export RELEASE_VERSION=$(echo $KUBE_VERSION | awk -F. '{print $1"."$2}')
+      curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz -O
+      tar -xvf run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz
+    else
+      curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${KUBE_VERSION}.tar.gz -O
+      tar -xvf run-${KUBE_VERSION}.tar.gz
+    fi
 }


@@ -63,8 +76,8 @@ source_file() {
    if [[ -e "pkg/run/$KUBE_VERSION/kubernetes.sh" ]]; then
      source pkg/run/$KUBE_VERSION/kubernetes.sh --role source
      install_pkg
-    elif [[ -e "pkg/run/$KUBE_VERSION/bin/kubernetes.sh" ]]; then
-      ROLE=deploy-nodes pkg/run/$KUBE_VERSION/bin/kubernetes.sh
+    elif [[ -e "pkg/run/$RELEASE_VERSION/bin/kubernetes.sh" ]]; then
+      ROLE=deploy-nodes pkg/run/$RELEASE_VERSION/bin/kubernetes.sh
    fi
 }

@@ -74,10 +87,36 @@ install_pkg() {
 }

 preset_gpu() {
-    GPU_PACKAGE_URL=http://aliacs-k8s-${REGION}.oss-${REGION}-internal.aliyuncs.com/public/pkg
+
+    if [[ $(echo "${KUBE_VERSION}" | cut -d. -f2) -lt 20 ]]; then
+      return
+    fi
+
    if [[ "$PRESET_GPU" == "true" ]]; then
-        bash -x pkg/run/$KUBE_VERSION/bin/nvidia-gpu-installer.sh --package-url-prefix ${GPU_PACKAGE_URL}
-        rm -rf /etc/kubernetes/manifests/nvidia-device-plugin.yml
+      for file_name in $(ls pkg/run/$RELEASE_VERSION/lib | grep -v init.sh); do
+          source pkg/run/$RELEASE_VERSION/lib/$file_name
+      done
+
+      if [[ $NVIDIA_DRIVER_VERSION == "" ]];then
+          export NVIDIA_DRIVER_VERSION=460.91.03
+      fi
+
+      nvidia::create_dir
+      # --nvidia-driver-runfile   指定驱动文件路径
+      nvidia::prepare_driver_package
+      # --nvidia-container-toolkit-rpms    指定nvidia container toolkit包含的rpm包所在目录
+      nvidia::prepare_container_runtime_package
+      # --nvidia-fabricmanager-rpm     指定nvidia fabric manager安装包（rpm格式）路径
+      nvidia::prepare_driver_package
+      # --nvidia-device-plugin-yaml     指定nvidia device plugin yaml文件路径
+      nvidia::deploy_static_pod
+
+      if [[ $RUNTIME == "docker" ]];then
+        export SKIP_CONTAINER_RUNTIME_CONFIG=true
+      fi
+
+      nvidia::gpu::installer::main
+
    fi
 }

@@ -194,15 +233,15 @@ pull_image() {
        systemctl start docker
        sleep 10

-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.2
+        docker pull registry-${REGION}-vpc.ack.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
+        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.5
        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.6.7
    else
        systemctl start containerd
        sleep 10

-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.2
+        ctr -n k8s.io i pull registry-${REGION}-vpc.ack.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
+        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.5
        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.6.7
    fi
 }
@@ -227,6 +266,12 @@ post_install() {
    fi
 }

+keep_container_data() {
+    if [[ "$KEEP_IMAGE_DATA" = "true" ]]; then
+        touch /var/.keep-container-data
+    fi
+}
+
 cleanup() {
    rm -rf ./{addon*,docker*,kubernetes*,pkg,run*}
 }
@@ -243,6 +288,7 @@ main() {
    source_file
    preset_gpu
    pull_image
+    keep_container_data
    update_os_release
    record_k8s_version
 }
--- a/scripts/ack-optimized-os-linux3-all.sh
+++ b/scripts/ack-optimized-os-linux3-all.sh
@@ -41,10 +41,13 @@ setup_env() {
    export OS="AliyunOS"

    if [[ "$RUNTIME" = "docker" ]]; then
-      export RUNTIME_VERSION="19.03.5"
-      export DOCKER_VERSION="19.03.5"
+      RUNTIME_VERSION=${RUNTIME_VERSION:-19.03.15}
+      export RUNTIME_VERSION
+      DOCKER_VERSION=${RUNTIME_VERSION:-19.03.15}
+      export DOCKER_VERSION
    else
-      export RUNTIME_VERSION="1.5.10"
+      RUNTIME_VERSION=${RUNTIME_VERSION:-1.6.20}
+      export RUNTIME_VERSION
    fi

    export RELEASE_VERSION=$(echo $KUBE_VERSION | awk -F. '{print $1"."$2}')
@@ -52,18 +55,29 @@ setup_env() {
    export REGION=$(curl --retry 10 -sSL http://100.100.100.200/latest/meta-data/region-id)
    export PKG_FILE_SERVER="http://aliacs-k8s-$REGION.oss-$REGION-internal.aliyuncs.com/$BETA_VERSION"
    export ACK_OPTIMIZED_OS_BUILD=1
+
+    # setup k8s pull image prefix
+    if [[ -z "$KUBE_REPO_PREFIX" && -n "$REGION" ]]; then
+      export KUBE_REPO_PREFIX=registry-vpc.$REGION.aliyuncs.com/acs
+    fi
 }


 download_pkg() {
-    curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz -O
-    tar -xvf run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz
+    if [[ $(echo "${KUBE_VERSION}" | cut -d. -f1) -ge 1 && $(echo "${KUBE_VERSION}" | cut -d. -f2) -ge 20 ]]; then
+      export RELEASE_VERSION=$(echo $KUBE_VERSION | awk -F. '{print $1"."$2}')
+      curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz -O
+      tar -xvf run-${RELEASE_VERSION}-linux-${OS_ARCH}.tar.gz
+    else
+      curl --retry 4 $PKG_FILE_SERVER/public/pkg/run/run-${KUBE_VERSION}.tar.gz -O
+      tar -xvf run-${KUBE_VERSION}.tar.gz
+    fi
 }


 source_file() {
-    if [[ -e "pkg/run/$RELEASE_VERSION/kubernetes.sh" ]]; then
-      source pkg/run/$RELEASE_VERSION/kubernetes.sh --role source
+    if [[ -e "pkg/run/$KUBE_VERSION/kubernetes.sh" ]]; then
+      source pkg/run/$KUBE_VERSION/kubernetes.sh --role source
      install_pkg
    elif [[ -e "pkg/run/$RELEASE_VERSION/bin/kubernetes.sh" ]]; then
      ROLE=deploy-nodes pkg/run/$RELEASE_VERSION/bin/kubernetes.sh
@@ -76,9 +90,36 @@ install_pkg() {
 }

 preset_gpu() {
-    GPU_PACKAGE_URL=http://aliacs-k8s-${REGION}.oss-${REGION}-internal.aliyuncs.com/public/pkg
+
+    if [[ $(echo "${KUBE_VERSION}" | cut -d. -f2) -lt 20 ]]; then
+      return
+    fi
+
    if [[ "$PRESET_GPU" == "true" ]]; then
-        bash -x pkg/run/$RELEASE_VERSION/bin/nvidia-gpu-installer.sh --package-url-prefix ${GPU_PACKAGE_URL}
+      for file_name in $(ls pkg/run/$RELEASE_VERSION/lib | grep -v init.sh); do
+          source pkg/run/$RELEASE_VERSION/lib/$file_name
+      done
+
+      if [[ $NVIDIA_DRIVER_VERSION == "" ]];then
+          export NVIDIA_DRIVER_VERSION=460.91.03
+      fi
+
+      nvidia::create_dir
+      # --nvidia-driver-runfile   指定驱动文件路径
+      nvidia::prepare_driver_package
+      # --nvidia-container-toolkit-rpms    指定nvidia container toolkit包含的rpm包所在目录
+      nvidia::prepare_container_runtime_package
+      # --nvidia-fabricmanager-rpm     指定nvidia fabric manager安装包（rpm格式）路径
+      nvidia::prepare_driver_package
+      # --nvidia-device-plugin-yaml     指定nvidia device plugin yaml文件路径
+      nvidia::deploy_static_pod
+
+      if [[ $RUNTIME == "docker" ]];then
+        export SKIP_CONTAINER_RUNTIME_CONFIG=true
+      fi
+
+      nvidia::gpu::installer::main
+
    fi
 }

@@ -188,15 +229,15 @@ pull_image() {
        systemctl start docker
        sleep 10

-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
-        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.2
+        docker pull registry-${REGION}-vpc.ack.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
+        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.5
        docker pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.6.7
    else
        systemctl start containerd
        sleep 10

-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
-        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.2
+        ctr -n k8s.io i pull registry-${REGION}-vpc.ack.aliyuncs.com/acs/kube-proxy:v${KUBE_VERSION}
+        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/pause:3.5
        ctr -n k8s.io i pull registry-vpc.${REGION}.aliyuncs.com/acs/coredns:1.6.7
    fi
 }
@@ -221,6 +262,12 @@ post_install() {
    fi
 }

+keep_container_data() {
+    if [[ "$KEEP_IMAGE_DATA" = "true" ]]; then
+        touch /var/.keep-container-data
+    fi
+}
+
 cleanup() {
    rm -rf ./{addon*,docker*,kubernetes*,pkg,run*}
 }
@@ -237,6 +284,7 @@ main() {
    source_file
    preset_gpu
    pull_image
+    keep_container_data
    update_os_release
    record_k8s_version
 }