Add all-in-one installation script

Signed-off-by: llhuii <liulinghui@huawei.com>

scripts/installation/all-in-one.sh

@@ -0,0 +1,494 @@
+#!/bin/bash
+
+# Copyright 2021 The KubeEdge Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script installs a all-in-one Sedna environment, including:
+# - A Kubernetes v1.21 cluster with multi worker nodes, default none worker node.
+# - KubeEdge with multi nodes, default is latest KubeEdge and one edge node.
+# - Sedna, default is latest release version.
+# 
+# It requires you:
+# - 2 CPUs or more
+# - 2GB+ free memory, depends on node number setting
+# - 10GB+ free disk space
+# - Internet connection(docker hub, github etc.)
+# - Linux platform, such as ubuntu/centos
+# - Docker 17.06+
+#
+# Advanced options, influential env vars:
+#
+# NUM_CLOUD_WORKERS   | optional | The cloud workers, default 0
+# NUM_EDGE_WORKERS    | optional | The KubeEdge workers, default 1
+# KUBEEDGE_VERSION   | optional | The KubeEdge version to be installed.
+#                              if not specified, it will get latest release version.
+# CLUSTER_NAME      | optional | The all-in-one cluster name, default 'sedna-mini'
+# FORCE_INSTALL_SEDNA      | optional | If 'true', force reinstall Sedna, default false.
+# NODE_IMAGE      | optional | Custom node image
+# REUSE_EDGE_CONTAINER      | optional | Whether reuse edge node containers or not, default is true
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+
+function prepare_env() {
+  : ${CLUSTER_NAME:=sedna-mini}
+
+  # here not use := because it ignore the error of get_latest_version command
+  if [ -z "${KUBEEDGE_VERSION:-}" ]; then
+    KUBEEDGE_VERSION=$(get_latest_version kubeedge/kubeedge v1.8.0)
+  fi
+
+  : ${NUM_CLOUD_WORKERS:=0}
+  : ${NUM_EDGE_WORKERS:=1}
+
+  # just reuse kind base image
+  # https://github.com/kubernetes-sigs/kind/blob/4910c3e221a858e68e29f9494170a38e1c4e8b80/pkg/build/nodeimage/defaults.go#L23
+
+  : ${ALLINONE_NODE_IMAGE:=kubeedge/sedna-allinone-node:v1.21.1}
+
+  readonly MAX_CLOUD_WORKERS=2
+  readonly MAX_EDGE_WORKERS=3
+
+  # TODO: find a better way to figure this kind control plane
+  readonly CONTROL_PLANE_NAME=${CLUSTER_NAME}-control-plane 
+
+  # cloudcore default websocket port 
+  : ${CLOUDCORE_WS_PORT:=10000}
+  # cloudcore default cert port 
+  : ${CLOUDCORE_CERT_PORT:=10002}
+
+  # for debug purpose
+  : ${RETAIN_CONTAINER:=}
+
+  # use existing edge node containers
+  # default is true
+  : ${REUSE_EDGE_CONTAINER:=true}
+
+  # force install sedna control plane
+  # default is false
+  : ${FORCE_INSTALL_SEDNA:=false}
+
+  # The docker network for edge nodes to separate the network of control plane.
+  # Since `kind` CNI doesn't support edge node, here just use the network 'kind'.
+  # TODO(llhuii): find a way to use the default docker network 'bridge'.
+  : ${EDGE_NODE_NETWORK:=kind}
+
+
+  validate_env
+}
+
+function validate_env() {
+
+  ((NUM_CLOUD_WORKERS<=MAX_CLOUD_WORKERS)) || {
+    log_fault "Only support NUM_CLOUD_WORKERS at most $MAX_CLOUD_WORKERS"
+  }
+
+  ((NUM_EDGE_WORKERS<=MAX_EDGE_WORKERS)) || {
+    log_fault "Only support NUM_EDGE_WORKERS at most $MAX_EDGE_WORKERS"
+  }
+}
+
+
+function _log() {
+  local level=$1
+  shift
+  timestamp=$(date +"[$level%m%d %H:%M:%S.%3N]")
+  echo "$timestamp $@"
+}
+
+function log_fault() {
+  _log E "$@" >&2
+  exit 2
+}
+
+function log_error() {
+  _log E "$@" >&2
+}
+
+function log_info() {
+  _log I "$@"
+}
+
+function gen_kind_config() {
+  cat <<EOF
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+name: $CLUSTER_NAME
+nodes:
+  - role: control-plane
+    image: $ALLINONE_NODE_IMAGE
+    # expose kubeedge cloudcore
+    extraPortMappings:
+    - containerPort: $CLOUDCORE_WS_PORT
+    - containerPort: $CLOUDCORE_CERT_PORT
+EOF
+
+  for((i=0;i<NUM_CLOUD_WORKERS;i++)); do
+  cat <<EOF
+  - role: worker
+    image: $ALLINONE_NODE_IMAGE
+EOF
+  done
+}
+
+function patch_kindnet() {
+  # Since in edge node, we just use containerd instead of docker, this requires CNI,
+  # And `kindnet` is the CNI in kind, requires `InClusterConfig`
+  # which would require KUBERNETES_SERVICE_HOST/KUBERNETES_SERVICE_PORT environment variables.
+  # But edgecore(up to 1.8.0) does not inject these environments.
+  # Here make a patch: can be any value
+  run_in_control_plane kubectl set env -n kube-system daemonset/kindnet KUBERNETES_SERVICE_HOST=10.96.0.1 KUBERNETES_SERVICE_PORT=443 
+}
+
+function create_k8s_cluster() {
+  if kind get clusters | grep -qx -F "$CLUSTER_NAME"; then
+    log_info "The k8s cluster $CLUSTER_NAME already exists, and just use it!"
+    log_info "If you want to recreate one, just run \`$0 clean\`."
+    return
+  fi
+
+  local extra_options=(--wait 90s)
+  [ -n "$RETAIN_CONTAINER" ] && extra_options+=(--retain)
+  gen_kind_config | kind create cluster ${extra_options[@]} --config -
+
+}
+
+function clean_k8s_cluster() {
+  kind delete cluster --name ${CLUSTER_NAME}
+}
+
+function run_in_control_plane() {
+  docker exec -i $CONTROL_PLANE_NAME "$@"
+}
+
+function get_control_plane_ip() {
+  # https://stackoverflow.com/a/20686101
+  docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $CONTROL_PLANE_NAME
+}
+
+function get_control_plane_exposed_port() {
+  local container_port=$1
+  docker inspect -f "{{(index (index .NetworkSettings.Ports \"${container_port}/tcp\") 0).HostPort}}" $CONTROL_PLANE_NAME
+}
+
+function setup_control_kubeconfig() {
+  run_in_control_plane bash -euc "
+    # copy kube config file
+    mkdir -p ~/.kube
+    cp /etc/kubernetes/admin.conf ~/.kube/config
+  "
+}
+
+function setup_cloudcore() {
+  # keadm already built into control plane
+
+  CLOUDCORE_LOCAL_IP=$(get_control_plane_ip)
+
+  # Use default docker network for edge nodes to separate the network of control plane which uses the defined network 'kind'
+  CLOUDCORE_EXPOSED_IP=$(get_docker_network_gw $EDGE_NODE_NETWORK)
+
+  CLOUDCORE_EXPOSED_WS_PORT=$(get_control_plane_exposed_port $CLOUDCORE_WS_PORT)
+  CLOUDCORE_EXPOSED_CERT_PORT=$(get_control_plane_exposed_port $CLOUDCORE_CERT_PORT)
+  CLOUDCORE_ADVERTISE_ADDRESSES=$CLOUDCORE_LOCAL_IP,$CLOUDCORE_EXPOSED_IP
+  CLOUDCORE_EXPOSED_ADDR=$CLOUDCORE_EXPOSED_IP:$CLOUDCORE_EXPOSED_WS_PORT
+
+  # keadm accepts version format: 1.8.0
+  local version=${KUBEEDGE_VERSION/v} 
+  run_in_control_plane bash -euc "
+    # install cloudcore
+    pgrep cloudcore >/dev/null || keadm init --kubeedge-version=$version --advertise-address=$CLOUDCORE_ADVERTISE_ADDRESSES"'
+
+    # wait token to be created
+    exit_code=1
+    TIMEOUT=30 # in seconds
+    for((i=1;i<=TIMEOUT; i++)); do
+      keadm gettoken >/dev/null 2>&1 && exit_code=0 && break
+      echo -ne "Waiting cloudcore to generate token, $i seconds...\r"
+      sleep 1
+    done
+    echo
+    if [ $exit_code -gt 0 ]; then
+      log_lines=50
+      tail -$log_lines /var/log/kubeedge/cloudcore.log | sed "s/^/    /"
+      echo "Timeout to wait cloudcore, above are the last $log_lines log of cloudcore."
+    fi
+    exit $exit_code
+  '
+  KUBEEDGE_TOKEN=$(run_in_control_plane keadm gettoken)
+}
+
+function gen_cni_config() {
+  cat <<EOF
+{
+  "cniVersion": "0.3.1",
+  "name": "edgecni",
+  "plugins": [
+    {
+      "type": "ptp",
+      "ipMasq": false,
+      "ipam": {
+        "type": "host-local",
+        "dataDir": "/run/cni-ipam-state",
+        "routes": [
+          {
+            "dst": "0.0.0.0/0"
+          }
+        ],
+        "ranges": [
+          [
+            {
+              "subnet": "10.244.0.0/24"
+            }
+          ]
+        ]
+      },
+      "mtu": 1500
+    },
+    {
+      "type": "portmap",
+      "capabilities": {
+        "portMappings": true
+      }
+    }
+  ]
+}
+
+EOF
+}
+
+function create_and_setup_edgenodes() {
+
+  for((i=0;i<NUM_EDGE_WORKERS;i++)); do
+    log_info "Installing $i-th edge node..."
+    local containername=sedna-mini-edge$i
+    local hostname=edge$i
+    local label=sedna.io=sedna-mini-edge
+
+    # Many tricky arguments are from the kind code
+    # https://github.com/kubernetes-sigs/kind/blob/4910c3e221a858e68e29f9494170a38e1c4e8b80/pkg/cluster/internal/providers/docker/provision.go#L148
+    local run_cmds=(
+      docker run
+      --network "$EDGE_NODE_NETWORK"
+      --hostname "$hostname"
+      --name "$containername"
+      --label $label
+      --privileged
+      --security-opt seccomp=unconfined
+      --security-opt apparmor=unconfined
+      --tmpfs /tmp
+      --tmpfs /run
+      --volume /var
+      # some k8s things want to read /lib/modules
+      --volume /lib/modules:/lib/modules:ro
+      --restart=on-failure:1
+      --tty
+      --detach $ALLINONE_NODE_IMAGE
+    )
+
+    local existing_id=$(docker ps -qa --filter name=$containername --filter label=$label)
+    if [ -n "$existing_id" ]; then
+      if [ "${REUSE_EDGE_CONTAINER,,}" = true ] ; then
+        log_info "Use existing container for ''$containername'"
+        log_info "If not your attention, you can do:"
+        log_info "  1) set REUSE_EDGE_CONTAINER=false"
+        log_info "  Or 2) clean it first."
+        log_info "And rerun this script."
+        # start in case stopped
+        docker start $containername
+      else
+        log_error "The container named $containername already exists, you can do:"
+        log_error "  1) set REUSE_EDGE_CONTAINER=true"
+        log_error "  Or 2) clean it first."
+        log_fault "And rerun this script."
+      fi
+    else
+      # does not exist, create one container for this edge
+      "${run_cmds[@]}"
+    fi
+
+    # install edgecore using keadm join
+    local version=${KUBEEDGE_VERSION/v}
+    docker exec -i $containername bash -uec "
+      pgrep edgecore >/dev/null || {
+        keadm join \
+          --cloudcore-ipport=${CLOUDCORE_EXPOSED_ADDR} \
+          --certport=${CLOUDCORE_EXPOSED_CERT_PORT} \
+          --token=$KUBEEDGE_TOKEN \
+          --kubeedge-version '$version' \
+          --edgenode-name '$hostname' \
+          --remote-runtime-endpoint unix:///var/run/containerd/containerd.sock \
+          --runtimetype remote
+
+        # set imageGCHighThreshold to 100% for no image gc
+        sed -i 's/imageGCHighThreshold:.*/imageGCHighThreshold: 100/' /etc/kubeedge/config/edgecore.yaml &&
+          systemctl restart edgecore ||
+          true  # ignore the error
+     }
+
+    "
+    gen_cni_config | docker exec -i $containername tee /etc/cni/net.d/10-edgecni.conflist >/dev/null
+   
+  done
+}
+
+function clean_edgenodes() {
+  for cid in $(docker ps -a --filter label=sedna.io=sedna-mini-edge -q); do 
+    docker stop $cid; docker rm $cid
+  done
+}
+
+function get_docker_network_gw() {
+  docker network inspect ${1-bridge} --format='{{(index .IPAM.Config 0).Gateway}}'
+}
+
+function setup_cloud() {
+  create_k8s_cluster
+
+  patch_kindnet
+
+  setup_control_kubeconfig
+
+  setup_cloudcore
+}
+
+function clean_cloud() {
+  clean_k8s_cluster
+}
+
+function setup_edge() {
+  create_and_setup_edgenodes
+}
+
+function clean_edge() {
+  clean_edgenodes
+}
+
+function install_sedna() {
+  local gm_node=$CONTROL_PLANE_NAME
+  if run_in_control_plane kubectl get ns sedna; then
+    if [ "$FORCE_INSTALL_SEDNA" != true ]; then
+      log_info '"sedna" namespace already exists, no install Sedna control components.'
+      log_info 'If want to reinstall them, you can remove it `kubectl delete ns sedna` or set FORCE_INSTALL_SEDNA=true!'
+      log_info
+      return
+    fi
+    run_in_control_plane bash -ec "
+    curl https://raw.githubusercontent.com/kubeedge/sedna/main/scripts/installation/install.sh | SEDNA_GM_NODE=$gm_node SEDNA_ACTION=clean bash -
+  "
+  fi
+
+  log_info "Installing Sedna Control Components..."
+
+  run_in_control_plane bash -ec "
+    curl https://raw.githubusercontent.com/kubeedge/sedna/main/scripts/installation/install.sh | SEDNA_GM_NODE=$gm_node SEDNA_ACTION=create bash -
+  "
+}
+
+function get_latest_version() {
+  # get the latest version of specified gh repo
+  local repo=${1} default_version=${2}
+  # output of this latest page:
+  # ...
+  # "tag_name": "v1.0.0",
+  # ...
+  
+  # Sometimes this will reach rate limit
+  # https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting
+  local url=https://api.github.com/repos/$repo/releases/latest
+  if ! curl --fail -s $url | awk '/"tag_name":/&&$0=$2' | sed 's/[",]//g'; then
+    log_error "Error to get latest version of $repo: $(curl -s $url | head)"
+    log_error "Fall back to default version: $default_version"
+    echo $default_version
+  fi
+}
+
+function arch() {
+  local arch=$(uname -m)
+  case "$arch" in
+    x86_64) arch=amd64;;
+    *);;
+  esac
+  echo "$arch"
+}
+
+function _download_tool() {
+  local name=$1 url=$2
+  local file=/usr/local/bin/$name 
+	curl -Lo $file $url
+  chmod +x $file
+}
+
+function check_command_exists() {
+  type $1 >/dev/null 2>&1
+}
+
+function ensure_tool() {
+  local command=$1 download_url=$2
+  if check_command_exists $command; then
+    return
+  fi
+
+	_download_tool $command $download_url
+
+}
+
+function ensure_kind() {
+  local version=${KIND_VERSION:-0.11.1}
+	ensure_tool kind https://kind.sigs.k8s.io/dl/v${version/v}/kind-linux-$(arch)
+}
+
+function ensure_kubectl() {
+
+  local version=${KUBECTL_VERSION:-1.21.0}
+  ensure_tool kubectl https://dl.k8s.io/release/v${version/v}/bin/linux/$(arch)/kubectl
+}
+
+function ensure_tools() {
+  ensure_kind
+  ensure_kubectl
+}
+
+function main() {
+  ensure_tools
+  prepare_env
+  action=${1-create}
+
+  case "$action" in
+    create)
+      # llh
+      setup_cloud
+      setup_edge
+      install_sedna
+      log_info "Mini Sedna is created successfully"
+      ;;
+
+    delete|clean)
+      clean_edge
+      clean_cloud
+      log_info "Mini Sedna is uninstalled successfully"
+      ;;
+
+    # As a source file, noop
+    __source__)
+      ;;
+
+    *)
+      log_fault "Unknown action $action"
+      ;;
+  esac
+}
+
+main "$@"

scripts/installation/build-allinone-image.sh

@@ -0,0 +1,155 @@
+#!/bin/bash
+
+# Copyright 2021 The KubeEdge Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script builds the node image for all-in-one Sedna.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+
+# just reuse kind image
+# https://github.com/kubernetes-sigs/kind/blob/4910c3e221a858e68e29f9494170a38e1c4e8b80/pkg/build/nodeimage/defaults.go#L23
+# 
+# Note: here use v1.21.1 of kindest/node, because kubeedge-1.8.0 still uses apiextensions.k8s.io/v1beta1 of CRD, which will be removed in k8s 1.22.0
+readonly BASE_IMAGE=kindest/node:v1.21.1
+
+readonly BUILD_IMAGE_NAME=sedna-edge-build-$RANDOM
+
+function get_latest_version() {
+  # get the latest version of specified gh repo
+  local repo=${1}
+  # output of this latest page:
+  # ...
+  # "tag_name": "v1.0.0",
+  # ...
+  curl -s https://api.github.com/repos/$repo/releases/latest | awk '/"tag_name":/&&$0=$2' | sed 's/[",]//g'
+}
+
+function create_build_container() {
+  docker run --rm --name $BUILD_IMAGE_NAME -d --entrypoint sleep $BASE_IMAGE inf || true
+
+  if [ -z "$RETAIN_BUILD_CONTAINER" ]; then
+    trap clean_build_container EXIT
+  fi
+}
+
+function clean_build_container() {
+  docker stop $BUILD_IMAGE_NAME
+}
+
+function run_in_build_container() {
+  docker exec -i $BUILD_IMAGE_NAME "$@"
+}
+
+function commit_build_container() {
+  local commit_args=(
+    docker commit
+    # put entrypoint back
+    # https://github.com/kubernetes-sigs/kind/blob/4910c3e221a858e68e29f9494170a38e1c4e8b80/images/base/Dockerfile#L203
+    --change 'ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]'
+    $BUILD_IMAGE_NAME $ALLINONE_NODE_IMAGE
+  )
+  "${commit_args[@]}"
+}
+
+function gen_edgecore_config_template() {
+  run_in_build_container mkdir -p /etc/kubeedge
+  cat | run_in_build_container mkdir -p /etc/kubeedge
+}
+
+function arch() {
+  local arch=$(uname -m)
+  case "$arch" in
+    x86_64) arch=amd64;;
+    *);;
+  esac
+  echo "$arch"
+}
+
+function install_keadm() {
+  # download the specified keadm binary
+  local arch=$(arch) version=${KUBEEDGE_VERSION}
+  local tarfile=keadm-$version-linux-$arch.tar.gz 
+  local path=keadm-$version-linux-$arch/keadm/keadm
+  local configdir=/etc/kubeedge
+
+  run_in_build_container bash -euc "
+    # copy kube config file
+    # install keadm
+    curl --fail -LSO https://github.com/kubeedge/kubeedge/releases/download/$version/$tarfile
+    tar -xvf $tarfile $path
+    mv $path /usr/local/bin/
+    rm $tarfile
+
+    # install dependencies of keadm init/join
+    apt update -y
+    apt install -y wget sudo mosquitto
+    systemctl enable mosquitto
+
+    # install debug tools
+    apt install -y less sqlite3
+
+    # add convenient command
+    echo 'alias docker=crictl' > ~/.bash_aliases
+  "
+  
+  # download the kubeedge into the docker image in advance
+  download_kubeedge
+}
+
+function download_kubeedge() {
+  # download the specified kubeedge package for keadm
+  local arch=$(arch) version=${KUBEEDGE_VERSION}
+  local tarfile=kubeedge-$version-linux-$arch.tar.gz 
+  local configdir=/etc/kubeedge
+
+  run_in_build_container bash -euc "
+    mkdir -p $configdir; cd $configdir
+    curl --fail -LSO https://github.com/kubeedge/kubeedge/releases/download/$version/$tarfile
+  "
+}
+
+function install_edgecore() {
+  # download the specified edgecore binary
+  local arch=$(arch) version=${KUBEEDGE_VERSION}
+  local tarfile=kubeedge-$version-linux-$arch.tar.gz 
+  local edgecorepath=kubeedge-$version-linux-$arch/edge/edgecore
+  local configdir=/etc/kubeedge
+
+  run_in_build_container bash -euc "
+    mkdir -p $configdir; cd $configdir
+    curl --fail -LSO https://github.com/kubeedge/kubeedge/releases/download/$version/$tarfile
+
+    tar -xvf $tarfile $edgecorepath
+    mv $edgecorepath /usr/local/bin/
+    rm $tarfile
+
+    # install service
+    curl --fail -LSO https://raw.githubusercontent.com/kubeedge/kubeedge/$version/build/tools/edgecore.service
+    systemctl enable $configdir/edgecore.service
+  "
+}
+
+: ${KUBEEDGE_VERSION:=$(get_latest_version kubeedge/kubeedge)}
+
+: ${NODE_IMAGE:=kubeedge/sedna-allinone-node:v1.21.1}
+: ${RETAIN_BUILD_CONTAINER:=}
+
+create_build_container
+install_keadm
+commit_build_container
+