feat(epctl): add upgrade restart controls and rollback-safe deploy

2026-07-07 02:06:16 +00:00 · 2026-06-29 22:55:36 +08:00
parent 33b39e9771
commit 6589ad10a6
4 changed files with 445 additions and 106 deletions
@@ -2,9 +2,8 @@
 set -euo pipefail

 readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly IMAGE_TAG="epctl-ubuntu-systemd-test:$(date +%s)-$$"
 readonly CONTAINER_NAME="epctl-systemd-test-$$"
-readonly BASE_IMAGE="ubuntu:24.04"
+readonly BASE_IMAGE="${EPCTL_DOCKER_IMAGE:-ubuntu:24.04}"

 ACTIVE_LANG=""
 POSITIONAL_ARGS=()
@@ -80,10 +79,8 @@ trf() {
    zh:upgrade_tag_invalid) template='升级 tag 必须是真实的 GitHub release tag，例如 v1.0.6' ;;
    en:base_image) template='base image: %s' ;;
    zh:base_image) template='基础镜像：%s' ;;
-    en:building_image) template='building the Ubuntu systemd test image' ;;
-    zh:building_image) template='正在构建 Ubuntu systemd 测试镜像' ;;
-    en:starting_container) template='starting the test container' ;;
-    zh:starting_container) template='正在启动测试容器' ;;
+    en:starting_container) template='starting the Ubuntu systemd test container directly from %s' ;;
+    zh:starting_container) template='正在直接从 %s 启动 Ubuntu systemd 测试容器' ;;
    en:systemd_not_ready) template='systemd did not become ready inside the container' ;;
    zh:systemd_not_ready) template='容器内的 systemd 未能及时就绪' ;;
    en:self_installing) template='installing epctl into PATH' ;;
@@ -96,6 +93,14 @@ trf() {
    zh:waiting_assets) template='正在等待 epusdt 启动并释放 www 静态文件' ;;
    en:upgrading_release) template='upgrading epusdt from %s to %s' ;;
    zh:upgrading_release) template='正在将 epusdt 从 %s 升级到 %s' ;;
+    en:verifying_upgrade_skip_restart) template='verifying that `upgrade --no-restart` skips the restart and keeps the current process running' ;;
+    zh:verifying_upgrade_skip_restart) template='正在验证 `upgrade --no-restart` 会跳过重启，并保持当前进程继续运行' ;;
+    en:verifying_upgrade_default_restart) template='verifying that a non-interactive upgrade restarts the service by default' ;;
+    zh:verifying_upgrade_default_restart) template='正在验证非交互升级默认会重启服务' ;;
+    en:verifying_upgrade_prompt_skip) template='verifying that `upgrade --prompt-restart` can skip the restart when `n` is entered' ;;
+    zh:verifying_upgrade_prompt_skip) template='正在验证 `upgrade --prompt-restart` 在输入 `n` 时会跳过重启' ;;
+    en:verifying_upgrade_prompt_restart) template='verifying that `upgrade --prompt-restart` restarts the service by default when Enter is pressed' ;;
+    zh:verifying_upgrade_prompt_restart) template='正在验证 `upgrade --prompt-restart` 在直接回车时默认会重启服务' ;;
    en:checking_init_password) template='checking init-password behavior' ;;
    zh:checking_init_password) template='正在检查 init-password 行为' ;;
    en:expected_second_failure) template='expected init-password to fail after the password was changed' ;;
@@ -181,25 +186,24 @@ map_arch() {
 }

 docker_bash() {
-  docker exec "${CONTAINER_NAME}" bash -lc "$1"
+  docker exec "${CONTAINER_NAME}" bash -euo pipefail -c "$1"
 }

 cleanup() {
  local exit_code="$?"
  if [[ "${exit_code}" -ne 0 ]]; then
    log "$(trf test_failed_collecting)"
-    docker exec "${CONTAINER_NAME}" bash -lc 'systemctl status epusdt --no-pager || true' || true
-    docker exec "${CONTAINER_NAME}" bash -lc 'journalctl -u epusdt --no-pager -n 100 || true' || true
+    docker logs "${CONTAINER_NAME}" >/dev/null 2>&1 && docker logs "${CONTAINER_NAME}" >&2 || true
+    docker exec "${CONTAINER_NAME}" bash -euo pipefail -c 'systemctl status epusdt --no-pager || true' || true
+    docker exec "${CONTAINER_NAME}" bash -euo pipefail -c 'journalctl -u epusdt --no-pager -n 100 || true' || true
  fi
  docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true
-  docker rmi -f "${IMAGE_TAG}" >/dev/null 2>&1 || true
-  rm -rf "${TMPDIR_PATH:-}"
 }

 wait_for_systemd() {
  local attempt
-  for attempt in $(seq 1 30); do
-    if docker exec "${CONTAINER_NAME}" bash -lc 'systemctl show --property=Version --value >/dev/null 2>&1'; then
+  for attempt in $(seq 1 120); do
+    if docker exec "${CONTAINER_NAME}" bash -euo pipefail -c 'systemctl show --property=Version --value >/dev/null 2>&1'; then
      return 0
    fi
    sleep 1
@@ -207,6 +211,24 @@ wait_for_systemd() {
  return 1
 }

+wait_for_service_assets() {
+  docker exec "${CONTAINER_NAME}" bash -euo pipefail -c '
+for _ in $(seq 1 30); do
+  if systemctl is-active --quiet epusdt && [[ -f /opt/epusdt/www/index.html ]]; then
+    exit 0
+  fi
+  sleep 1
+done
+systemctl status epusdt --no-pager || true
+journalctl -u epusdt --no-pager -n 100 || true
+exit 1
+'
+}
+
+service_main_pid() {
+  docker exec "${CONTAINER_NAME}" bash -euo pipefail -c 'systemctl show --property=MainPID --value epusdt' | tr -d '\r\n'
+}
+
 set_language ""
 parse_global_options "$@"

@@ -235,39 +257,10 @@ if [[ -n "${UPGRADE_TAG}" ]]; then
  curl --fail --silent --show-error --head "https://github.com/GMWalletApp/epusdt/releases/download/${UPGRADE_TAG}/SHA256SUMS" >/dev/null
 fi

-TMPDIR_PATH="$(mktemp -d)"
 trap cleanup EXIT

-cat > "${TMPDIR_PATH}/Dockerfile" <<EOF
-FROM ${BASE_IMAGE}
-
-ENV container=docker
-
-RUN apt-get update \
-  && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    ca-certificates \
-    curl \
-    jq \
-    sudo \
-    systemd \
-    systemd-sysv \
-    tar \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/* \
-  && useradd -m -s /bin/bash tester \
-  && echo 'tester ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/tester \
-  && chmod 0440 /etc/sudoers.d/tester
-
-STOPSIGNAL SIGRTMIN+3
-
-CMD ["/sbin/init"]
-EOF
-
 log "$(trf base_image "${BASE_IMAGE}")"
-log "$(trf building_image)"
-docker build -t "${IMAGE_TAG}" "${TMPDIR_PATH}" >/dev/null
-
-log "$(trf starting_container)"
+log "$(trf starting_container "${BASE_IMAGE}")"
 docker run -d \
  --name "${CONTAINER_NAME}" \
  --privileged \
@@ -278,75 +271,152 @@ docker run -d \
  -v /sys/fs/cgroup:/sys/fs/cgroup:rw \
  -v "${SCRIPT_DIR}:/work" \
  -w /work \
-  "${IMAGE_TAG}" \
-  /sbin/init >/dev/null
+  "${BASE_IMAGE}" \
+  bash -euo pipefail -c '
+    export DEBIAN_FRONTEND=noninteractive
+    apt-get update
+    apt-get install -y \
+      ca-certificates \
+      curl \
+      expect \
+      jq \
+      sudo \
+      systemd \
+      systemd-sysv \
+      tar
+    apt-get clean
+    rm -rf /var/lib/apt/lists/*
+    if ! id -u tester >/dev/null 2>&1; then
+      useradd -m -s /bin/bash tester
+    fi
+    mkdir -p /etc/sudoers.d
+    echo "tester ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/tester
+    chmod 0440 /etc/sudoers.d/tester
+    exec /sbin/init
+  ' >/dev/null

 wait_for_systemd || die "$(trf systemd_not_ready)"

 log "$(trf self_installing)"
-docker exec "${CONTAINER_NAME}" bash -lc "su - tester -c 'cd /work && ./epctl self-install'"
+docker exec "${CONTAINER_NAME}" bash -euo pipefail -c "su - tester -c 'cd /work && ./epctl self-install'"
 docker_bash '[[ -x /usr/local/bin/epctl ]]'
-docker exec "${CONTAINER_NAME}" bash -lc 'su - tester -c "EPCTL_NO_COLOR=1 epctl help | grep -q \"^用法：\""'
-docker exec "${CONTAINER_NAME}" bash -lc 'su - tester -c "EPCTL_NO_COLOR=1 epctl --lang zh help | grep -q \"^用法：\""'
-docker exec "${CONTAINER_NAME}" bash -lc 'su - tester -c "EPCTL_NO_COLOR=1 epctl --lang en help | grep -q \"^Usage:\""'
+docker exec "${CONTAINER_NAME}" bash -euo pipefail -c 'su - tester -c "EPCTL_NO_COLOR=1 epctl help | grep -q \"^用法：\""'
+docker exec "${CONTAINER_NAME}" bash -euo pipefail -c 'su - tester -c "EPCTL_NO_COLOR=1 epctl --lang zh help | grep -q \"^用法：\""'
+docker exec "${CONTAINER_NAME}" bash -euo pipefail -c 'su - tester -c "EPCTL_NO_COLOR=1 epctl --lang en help | grep -q \"^Usage:\""'

 log "$(trf downloading_release "${INSTALL_TAG}")"
-docker exec "${CONTAINER_NAME}" env TAG="${INSTALL_TAG}" bash -lc 'su - tester -c "epctl download --tag ${TAG}"'
+docker exec "${CONTAINER_NAME}" env TAG="${INSTALL_TAG}" bash -euo pipefail -c 'su - tester -c "epctl download --tag ${TAG}"'

 log "$(trf installing_release "${INSTALL_TAG}")"
-docker exec "${CONTAINER_NAME}" env TAG="${INSTALL_TAG}" bash -lc 'su - tester -c "epctl install --tag ${TAG} --app-uri http://127.0.0.1:8000"'
+docker exec "${CONTAINER_NAME}" env TAG="${INSTALL_TAG}" bash -euo pipefail -c 'su - tester -c "epctl install --tag ${TAG} --app-uri http://127.0.0.1:8000"'

 log "$(trf waiting_assets)"
-docker exec "${CONTAINER_NAME}" bash -lc '
-for _ in $(seq 1 30); do
-  if systemctl is-active --quiet epusdt && [[ -f /opt/epusdt/www/index.html ]]; then
-    exit 0
-  fi
-  sleep 1
-done
-systemctl status epusdt --no-pager || true
-journalctl -u epusdt --no-pager -n 100 || true
-exit 1
-'
+wait_for_service_assets

 docker_bash '[[ -x /opt/epusdt/epusdt ]]'
 docker_bash '[[ -f /opt/epusdt/.env ]]'
 docker_bash 'systemctl is-active --quiet epusdt'
 docker_bash '[[ -f /opt/epusdt/www/index.html ]]'
-docker exec "${CONTAINER_NAME}" bash -lc 'su - tester -c "epctl show-config | grep -q \"^install=false$\""'
-docker exec "${CONTAINER_NAME}" bash -lc 'su - tester -c "EPCTL_NO_COLOR=1 epctl --lang zh show-config | grep -q \"^install=false$\""'
-docker exec "${CONTAINER_NAME}" bash -lc '
+docker exec "${CONTAINER_NAME}" bash -euo pipefail -c 'su - tester -c "epctl show-config | grep -q \"^install=false$\""'
+docker exec "${CONTAINER_NAME}" bash -euo pipefail -c 'su - tester -c "EPCTL_NO_COLOR=1 epctl --lang zh show-config | grep -q \"^install=false$\""'
+docker exec "${CONTAINER_NAME}" bash -euo pipefail -c '
 su - tester -c "epctl status >/tmp/epctl-status.out 2>/tmp/epctl-status.err"
 ! grep -q "systemd-journal" /tmp/epctl-status.err
 '
-docker exec "${CONTAINER_NAME}" bash -lc '
+docker exec "${CONTAINER_NAME}" bash -euo pipefail -c '
 su - tester -c "epctl logs --lines 50 >/tmp/epctl-logs.out 2>/tmp/epctl-logs.err"
 ! grep -q "systemd-journal" /tmp/epctl-logs.err
 '

 if [[ -n "${UPGRADE_TAG}" ]]; then
  log "$(trf upgrading_release "${INSTALL_TAG}" "${UPGRADE_TAG}")"
-  docker exec "${CONTAINER_NAME}" env TAG="${UPGRADE_TAG}" bash -lc 'su - tester -c "epctl upgrade --tag ${TAG}"'
-  docker exec "${CONTAINER_NAME}" bash -lc '
-for _ in $(seq 1 30); do
-  if systemctl is-active --quiet epusdt && [[ -f /opt/epusdt/www/index.html ]]; then
-    exit 0
-  fi
-  sleep 1
-done
-systemctl status epusdt --no-pager || true
-journalctl -u epusdt --no-pager -n 100 || true
-exit 1
+  env_checksum_before="$(docker exec "${CONTAINER_NAME}" bash -euo pipefail -c 'sha256sum /opt/epusdt/.env | cut -d" " -f1' | tr -d '\r\n')"
+  pid_before_skip_restart="$(service_main_pid)"
+
+  log "$(trf verifying_upgrade_skip_restart)"
+  docker exec "${CONTAINER_NAME}" env TAG="${UPGRADE_TAG}" bash -euo pipefail -c '
+output_file="$(mktemp)"
+su - tester -c "EPCTL_NO_COLOR=1 epctl --lang en upgrade --tag ${TAG} --no-restart" >"${output_file}" 2>&1
+grep -q "systemctl restart epusdt" "${output_file}"
+! grep -q "Restart .*\[Y/n\]" "${output_file}"
+rm -f "${output_file}"
 '
-  docker exec "${CONTAINER_NAME}" bash -lc 'su - tester -c "epctl show-config | grep -q \"^install=false$\""'
-  docker exec "${CONTAINER_NAME}" bash -lc '
+  pid_after_skip_restart="$(service_main_pid)"
+  [[ "${pid_before_skip_restart}" == "${pid_after_skip_restart}" ]]
+  docker exec "${CONTAINER_NAME}" env TAG="${UPGRADE_TAG}" bash -euo pipefail -c '/opt/epusdt/epusdt version | grep -q "^version: ${TAG}$"'
+  docker exec "${CONTAINER_NAME}" env TAG="${UPGRADE_TAG}" bash -euo pipefail -c 'cmp -s /opt/epusdt/.env.example "/tmp/epusdt/${TAG}/extract/.env.example"'
+  env_checksum_after_skip="$(docker exec "${CONTAINER_NAME}" bash -euo pipefail -c 'sha256sum /opt/epusdt/.env | cut -d" " -f1' | tr -d '\r\n')"
+  [[ "${env_checksum_before}" == "${env_checksum_after_skip}" ]]
+
+  log "$(trf verifying_upgrade_default_restart)"
+  pid_before_default_restart="$(service_main_pid)"
+  docker exec "${CONTAINER_NAME}" env TAG="${UPGRADE_TAG}" bash -euo pipefail -c '
+output_file="$(mktemp)"
+su - tester -c "EPCTL_NO_COLOR=1 epctl --lang en upgrade --tag ${TAG}" >"${output_file}" 2>&1
+! grep -q "Restart .*\[Y/n\]" "${output_file}"
+rm -f "${output_file}"
+'
+  wait_for_service_assets
+  pid_after_default_restart="$(service_main_pid)"
+  [[ "${pid_before_default_restart}" != "${pid_after_default_restart}" ]]
+
+  docker exec "${CONTAINER_NAME}" env TAG="${UPGRADE_TAG}" bash -euo pipefail -c '
+output_file="$(mktemp)"
+if su - tester -c "EPCTL_NO_COLOR=1 epctl --lang en upgrade --tag ${TAG} --prompt-restart" </dev/null >"${output_file}" 2>&1; then
+  cat "${output_file}" >&2
+  rm -f "${output_file}"
+  exit 1
+fi
+grep -q "interactive terminal" "${output_file}"
+rm -f "${output_file}"
+'
+
+  log "$(trf verifying_upgrade_prompt_skip)"
+  pid_before_prompt_skip="$(service_main_pid)"
+  docker exec "${CONTAINER_NAME}" env TAG="${UPGRADE_TAG}" bash -euo pipefail -c '
+output_file="$(mktemp)"
+OUTPUT_FILE="${output_file}" expect <<'"'"'EOF'"'"'
+set timeout 120
+log_file -noappend $env(OUTPUT_FILE)
+spawn su - tester -c "EPCTL_NO_COLOR=1 epctl --lang en upgrade --tag $env(TAG) --prompt-restart"
+expect {
+  -re {Restart .*\[Y/n\]} { send "n\r" }
+  timeout { exit 1 }
+}
+expect eof
+EOF
+grep -q "systemctl restart epusdt" "${output_file}"
+rm -f "${output_file}"
+'
+  pid_after_prompt_skip="$(service_main_pid)"
+  [[ "${pid_before_prompt_skip}" == "${pid_after_prompt_skip}" ]]
+
+  log "$(trf verifying_upgrade_prompt_restart)"
+  pid_before_prompt_restart="$(service_main_pid)"
+  docker exec "${CONTAINER_NAME}" env TAG="${UPGRADE_TAG}" bash -euo pipefail -c '
+expect <<'"'"'EOF'"'"'
+set timeout 120
+spawn su - tester -c "EPCTL_NO_COLOR=1 epctl --lang en upgrade --tag $env(TAG) --prompt-restart"
+expect {
+  -re {Restart .*\[Y/n\]} { send "\r" }
+  timeout { exit 1 }
+}
+expect eof
+EOF
+'
+  wait_for_service_assets
+  pid_after_prompt_restart="$(service_main_pid)"
+  [[ "${pid_before_prompt_restart}" != "${pid_after_prompt_restart}" ]]
+
+  docker exec "${CONTAINER_NAME}" bash -euo pipefail -c 'su - tester -c "epctl show-config | grep -q \"^install=false$\""'
+  docker exec "${CONTAINER_NAME}" bash -euo pipefail -c '
 su - tester -c "epctl status >/tmp/epctl-status-upgrade.out 2>/tmp/epctl-status-upgrade.err"
 ! grep -q "systemd-journal" /tmp/epctl-status-upgrade.err
 '
 fi

 log "$(trf checking_init_password)"
-docker exec "${CONTAINER_NAME}" bash -lc '
+docker exec "${CONTAINER_NAME}" bash -euo pipefail -c '
 response="$(su - tester -c "epctl init-password")"
 password="$(printf "%s\n" "${response}" | jq -r ".data.password")"
 printf "%s\n" "${response}" | jq -e ".status_code == 200 and (.data.password | type == \"string\" and length > 0)" >/dev/null