diff --git a/scripts/installation/all-in-one.sh b/scripts/installation/all-in-one.sh index 8e352e85..281bc359 100755 --- a/scripts/installation/all-in-one.sh +++ b/scripts/installation/all-in-one.sh @@ -18,7 +18,7 @@ # - A Kubernetes v1.21 cluster with multi worker nodes, default none worker node. # - KubeEdge with multi nodes, default is latest KubeEdge and one edge node. # - Sedna, default is latest release version. -# +# # It requires you: # - 2 CPUs or more # - 2GB+ free memory, depends on node number setting @@ -76,12 +76,12 @@ function prepare_env() { readonly MAX_EDGE_WORKER_NODES=3 # TODO: find a better way to figure this kind control plane - readonly CONTROL_PLANE_NAME=${CLUSTER_NAME}-control-plane + readonly CONTROL_PLANE_NAME=${CLUSTER_NAME}-control-plane readonly CLOUD_WORKER_NODE_NAME=${CLUSTER_NAME}-worker - # cloudcore default websocket port + # cloudcore default websocket port : ${CLOUDCORE_WS_PORT:=10000} - # cloudcore default cert port + # cloudcore default cert port : ${CLOUDCORE_CERT_PORT:=10002} # for debug purpose @@ -168,7 +168,7 @@ function patch_kindnet() { # which would require KUBERNETES_SERVICE_HOST/KUBERNETES_SERVICE_PORT environment variables. # But edgecore(up to 1.8.0) does not inject these environments. # Here make a patch: can be any value - run_in_control_plane kubectl set env -n kube-system daemonset/kindnet KUBERNETES_SERVICE_HOST=10.96.0.1 KUBERNETES_SERVICE_PORT=443 + run_in_control_plane kubectl set env -n kube-system daemonset/kindnet KUBERNETES_SERVICE_HOST=10.96.0.1 KUBERNETES_SERVICE_PORT=443 } function create_k8s_cluster() { @@ -224,10 +224,15 @@ function setup_cloudcore() { CLOUDCORE_EXPOSED_ADDR=$CLOUDCORE_EXPOSED_IP:$CLOUDCORE_EXPOSED_WS_PORT # keadm accepts version format: 1.8.0 - local version=${KUBEEDGE_VERSION/v} + local version=${KUBEEDGE_VERSION/v} run_in_control_plane bash -euc " # install cloudcore - pgrep cloudcore >/dev/null || keadm init --kubeedge-version=$version --advertise-address=$CLOUDCORE_ADVERTISE_ADDRESSES"' + pgrep cloudcore >/dev/null || { + # keadm 1.8.1 is incompatible with 1.9.1 since crds' upgrade + rm -rf /etc/kubeedge/crds + + keadm init --kubeedge-version=$version --advertise-address=$CLOUDCORE_ADVERTISE_ADDRESSES"' + } # wait token to be created exit_code=1 @@ -248,10 +253,168 @@ function setup_cloudcore() { KUBEEDGE_TOKEN=$(run_in_control_plane keadm gettoken) } +_change_detect_yaml_change() { + # execute the specified yq commands on stdin + # if same, output nothing + # else output the updated yaml + local yq_cmds="${1:-.}" + docker run -i --rm --entrypoint sh mikefarah/yq -c " + yq e . - > a + yq e '$yq_cmds' a > b + cmp -s a b || cat b + " +} + +reconfigure_edgecore() { + # update edgecore.yaml for every edge node + local script_name=reconfigure-edgecore + + if ((NUM_EDGE_NODES<1)); then + return + fi + + local yq_cmds="$1" + + # I want to leverage kubectl but k8s has no ways to run job on each node once + # see https://github.com/kubernetes/kubernetes/issues/64623 for more detais + # So I use Daemonset temporarily + + kubectl apply -f - < a && yq e '$(echo $yq_cmds)' a > b || exit 1 + cmp -s a b && echo No need to reconfigure \$NODE_NAME edgecore || { + # backup and overwrite config, kill edgecore and wait systemd restart it + cp /config/edgecore.yaml /config/edgecore.yaml.reconfigure_bk + cp b /config/edgecore.yaml + + pkill edgecore + + # check to edgecore process status + > pids + for i in 0 1 2 3; do + sleep 10 + { pidof edgecore || echo =\$i; } >> pids + done + [ \$(sort -u pids | wc -l) -le 2 ] && echo Reconfigure \$NODE_NAME edgecore successfully || { + echo Failed to reconfigure \$NODE_NAME edgecore >&2 + echo And recovery edgecore config yaml >&2 + cp a /config/edgecore.yaml + + # prevent daemonset execute this script too frequently + sleep 1800 + exit 1 + } + } + sleep inf +EOF + + # wait this script been executed + kubectl -n kubeedge rollout status --timeout=5m ds $script_name + # wait all edge nodes to be ready if restarted + kubectl wait --for=condition=ready node -l node-role.kubernetes.io/edge= -function setup_edgemesh() { - # TODO: wait for edgemesh one line installer - : + # keep this daemonset script for debugging + # kubectl -n kubeedge delete ds $script_name + +} + +reconfigure_cloudcore() { + + local config_file=/etc/kubeedge/config/cloudcore.yaml + local yq_cmds=$1 + + run_in_control_plane cat $config_file | + _change_detect_yaml_change "$yq_cmds" | + run_in_control_plane bash -euc " + cat > cc.yaml + ! grep -q . cc.yaml || { + echo reconfigure and restart cloudcore + cp $config_file ${config_file}.reconfigure_bk + cp cc.yaml $config_file + pkill cloudcore || true + # TODO: use a systemd service + (cloudcore &>> /var/log/kubeedge/cloudcore.log &) + } + + " + echo Reconfigure cloudcore successfully +} + +function install_edgemesh() { + if ((NUM_EDGE_NODES<1)); then + # no edge node, no edgemesh + return + fi + + local server_node_name + if ((NUM_CLOUD_WORKER_NODES>0)); then + server_node_name=${CLUSTER_NAME}-worker + else + server_node_name=${CLUSTER_NAME}-control-plane + fi + + echo Installing edgemesh with server on $server_node_name + # enable Local APIServer + reconfigure_cloudcore '.modules.dynamicController.enable=true' + + reconfigure_edgecore ' + .modules.edged.clusterDNS="169.254.96.16" + | .modules.edged.clusterDomain="cluster.local" + | .modules.metaManager.metaServer.enable=true + ' + + # no server.publicIP + # since allinone is in flat network, we just use private ip for edgemesh server + helm upgrade --install edgemesh \ + --set server.nodeName=$server_node_name \ + https://raw.githubusercontent.com/kubeedge/edgemesh/main/build/helm/edgemesh.tgz + + echo Install edgemesh successfully } function gen_cni_config() { @@ -365,12 +528,24 @@ function create_and_setup_edgenodes() { " # fix cni config file gen_cni_config | docker exec -i $containername tee /etc/cni/net.d/10-edgecni.conflist >/dev/null - + + { + # wait edge node to be created at background + nwait=20 + for((i=0;i/dev/null && break + sleep 3 + done + } & + done + # wait all edge nodes to be created + wait + } function clean_edgenodes() { - for cid in $(docker ps -a --filter label=sedna.io=sedna-mini-edge -q); do + for cid in $(docker ps -a --filter label=sedna.io=sedna-mini-edge -q); do docker stop $cid; docker rm $cid done } @@ -387,8 +562,6 @@ function setup_cloud() { setup_control_kubeconfig setup_cloudcore - - setup_edgemesh } function clean_cloud() { @@ -434,7 +607,7 @@ function get_latest_version() { # ... # "tag_name": "v1.0.0", # ... - + # Sometimes this will reach rate limit # https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting local url=https://api.github.com/repos/$repo/releases/latest @@ -458,7 +631,7 @@ function arch() { function _download_tool() { local name=$1 url=$2 - local file=/usr/local/bin/$name + local file=/usr/local/bin/$name curl -Lo $file $url chmod +x $file } @@ -488,9 +661,17 @@ function ensure_kubectl() { ensure_tool kubectl https://dl.k8s.io/release/v${version/v}/bin/linux/$(arch)/kubectl } +function ensure_helm() { + if check_command_exists helm; then + return + fi + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +} + function ensure_tools() { ensure_kind ensure_kubectl + ensure_helm } function main() { @@ -502,6 +683,11 @@ function main() { create) setup_cloud setup_edge + # wait all nodes to be ready + kubectl wait --for=condition=ready node --all + + # edgemesh need to be installed before sedna + install_edgemesh install_sedna log_info "Mini Sedna is created successfully" ;;