在 PVE 上给软路由虚拟机加一个网络 Watchdog

1
#!/usr/bin/env bash
2
set -euo pipefail
3

4
: "${CONFIG_FILE:=/etc/default/pve-net-watchdog}"
5

6
set_defaults() {
7
  : "${VMID:=103}"
8
  : "${ROUTER_IP:=192.168.1.1}"
9
  : "${PING_TARGET:=www.baidu.com}"
10
  : "${PING_COUNT:=3}"
11
  : "${PING_TIMEOUT_SECONDS:=2}"
12
  : "${ROUTER_RESTART_AFTER_MINUTES:=5}"
13
  : "${ROUTER_COOLDOWN_MINUTES:=10}"
14
  : "${ROUTER_MAX_RESTARTS_BEFORE_PVE:=3}"
15
  : "${ENABLE_PVE_REBOOT:=0}"
16
  : "${PVE_REBOOT_AFTER_LAST_ROUTER_MINUTES:=5}"
17
  : "${PVE_REBOOT_COOLDOWN_HOURS:=24}"
18
  : "${STATE_DIR:=/var/lib/pve-net-watchdog}"
19
  : "${LOCK_FILE:=/run/pve-net-watchdog.lock}"
20
  : "${DRY_RUN:=0}"
21
}
22

23
load_config() {
24
  if [[ -r "$CONFIG_FILE" ]]; then
25
    # shellcheck source=/dev/null
26
    source "$CONFIG_FILE"
27
  fi
28
  set_defaults
29
  STATE_FILE="$STATE_DIR/state"
30
}
31

32
log() {
33
  local level="$1"
34
  shift
35
  local message="$*"
36
  printf '%s [%s] %s\n' "$(date -Is)" "$level" "$message"
37
  logger -t pve-net-watchdog "[$level] $message"
38
}
39

40
ping_host() {
41
  local host="$1"
42
  ping -c "$PING_COUNT" -W "$PING_TIMEOUT_SECONDS" "$host" >/dev/null 2>&1
43
}
44

45
run_cmd() {
46
  if [[ "$DRY_RUN" == "1" ]]; then
47
    log info "DRY_RUN would run: $*"
48
    return 0
49
  fi
50
  "$@"
51
}
52

53
init_state() {
54
  FAIL_COUNT=0
55
  FIRST_FAILURE_EPOCH=0
56
  ROUTER_RESTART_COUNT=0
57
  LAST_ROUTER_RESTART_EPOCH=0
58
  LAST_PVE_REBOOT_EPOCH=0
59
}
60

61
ensure_state_dir() {
62
  mkdir -p "$STATE_DIR"
63
  chmod 0700 "$STATE_DIR"
64
}
65

66
load_state() {
67
  init_state
68
  if [[ -r "$STATE_FILE" ]]; then
69
    # shellcheck source=/dev/null
70
    source "$STATE_FILE"
71
  fi
72
}
73

74
save_state() {
75
  local tmp="$STATE_FILE.tmp.$$"
76
  umask 077
77
  {
78
    printf 'FAIL_COUNT=%s\n' "$FAIL_COUNT"
79
    printf 'FIRST_FAILURE_EPOCH=%s\n' "$FIRST_FAILURE_EPOCH"
80
    printf 'ROUTER_RESTART_COUNT=%s\n' "$ROUTER_RESTART_COUNT"
81
    printf 'LAST_ROUTER_RESTART_EPOCH=%s\n' "$LAST_ROUTER_RESTART_EPOCH"
82
    printf 'LAST_PVE_REBOOT_EPOCH=%s\n' "$LAST_PVE_REBOOT_EPOCH"
83
  } >"$tmp"
84
  mv "$tmp" "$STATE_FILE"
85
}
86

87
elapsed_minutes_since() {
88
  local start_epoch="$1"
89
  local now="$2"
90
  if [[ "$start_epoch" -le 0 ]]; then
91
    echo 0
92
    return
93
  fi
94
  echo $(((now - start_epoch) / 60))
95
}
96

97
minutes_have_passed() {
98
  local last_epoch="$1"
99
  local now="$2"
100
  local minutes="$3"
101
  [[ "$last_epoch" -le 0 ]] && return 0
102
  [[ $((now - last_epoch)) -ge $((minutes * 60)) ]]
103
}
104

105
hours_have_passed() {
106
  local last_epoch="$1"
107
  local now="$2"
108
  local hours="$3"
109
  [[ "$last_epoch" -le 0 ]] && return 0
110
  [[ $((now - last_epoch)) -ge $((hours * 3600)) ]]
111
}
112

113
reset_failure_state() {
114
  FAIL_COUNT=0
115
  FIRST_FAILURE_EPOCH=0
116
  ROUTER_RESTART_COUNT=0
117
  LAST_ROUTER_RESTART_EPOCH=0
118
}
119

120
recover_router_vm() {
121
  local status
122
  status="$(qm status "$VMID" 2>/dev/null || true)"
123

124
  if ! grep -q 'status: running' <<<"$status"; then
125
    log warning "VM $VMID is not running; starting it"
126
    run_cmd qm start "$VMID"
127
    return
128
  fi
129

130
  log warning "VM $VMID is running but connectivity is still down; rebooting it"
131
  if ! run_cmd qm reboot "$VMID"; then
132
    log error "qm reboot $VMID failed; resetting VM"
133
    run_cmd qm reset "$VMID"
134
  fi
135
}
136

137
maybe_recover_router() {
138
  local now="$1"
139
  local failure_minutes="$2"
140

141
  if [[ "$failure_minutes" -lt "$ROUTER_RESTART_AFTER_MINUTES" ]]; then
142
    return
143
  fi
144

145
  if [[ "$ROUTER_RESTART_COUNT" -ge "$ROUTER_MAX_RESTARTS_BEFORE_PVE" ]]; then
146
    log warning "router restart limit reached: $ROUTER_RESTART_COUNT/$ROUTER_MAX_RESTARTS_BEFORE_PVE"
147
    return
148
  fi
149

150
  if ! minutes_have_passed "$LAST_ROUTER_RESTART_EPOCH" "$now" "$ROUTER_COOLDOWN_MINUTES"; then
151
    log warning "connectivity still down; router restart cooldown is active"
152
    return
153
  fi
154

155
  recover_router_vm
156
  ROUTER_RESTART_COUNT=$((ROUTER_RESTART_COUNT + 1))
157
  LAST_ROUTER_RESTART_EPOCH="$now"
158
  log warning "recorded router recovery attempt $ROUTER_RESTART_COUNT/$ROUTER_MAX_RESTARTS_BEFORE_PVE"
159
}
160

161
maybe_reboot_pve() {
162
  local now="$1"
163

164
  if [[ "$ENABLE_PVE_REBOOT" != "1" ]]; then
165
    return
166
  fi
167

168
  if [[ "$ROUTER_RESTART_COUNT" -lt "$ROUTER_MAX_RESTARTS_BEFORE_PVE" ]]; then
169
    return
170
  fi
171

172
  if ! minutes_have_passed "$LAST_ROUTER_RESTART_EPOCH" "$now" "$PVE_REBOOT_AFTER_LAST_ROUTER_MINUTES"; then
173
    log warning "PVE reboot threshold reached, waiting for post-router-restart grace period"
174
    return
175
  fi
176

177
  if ! hours_have_passed "$LAST_PVE_REBOOT_EPOCH" "$now" "$PVE_REBOOT_COOLDOWN_HOURS"; then
178
    log error "PVE reboot suppressed by cooldown"
179
    return
180
  fi
181

182
  LAST_PVE_REBOOT_EPOCH="$now"
183
  save_state
184
  log error "connectivity still down after $ROUTER_RESTART_COUNT router recoveries; rebooting PVE host"
185
  run_cmd systemctl reboot
186
}
187

188
run_once() {
189
  load_config
190
  ensure_state_dir
191
  load_state
192

193
  local now
194
  now="$(date +%s)"
195

196
  if ping_host "$PING_TARGET"; then
197
    if [[ "$FAIL_COUNT" -gt 0 || "$ROUTER_RESTART_COUNT" -gt 0 ]]; then
198
      log info "connectivity restored via $PING_TARGET; resetting failure state"
199
    fi
200
    reset_failure_state
201
    save_state
202
    return
203
  fi
204

205
  if [[ "$FIRST_FAILURE_EPOCH" -le 0 ]]; then
206
    FIRST_FAILURE_EPOCH="$now"
207
  fi
208
  FAIL_COUNT=$((FAIL_COUNT + 1))
209

210
  local failure_minutes
211
  failure_minutes="$(elapsed_minutes_since "$FIRST_FAILURE_EPOCH" "$now")"
212

213
  if ping_host "$ROUTER_IP"; then
214
    log warning "connectivity check failed for $PING_TARGET; router $ROUTER_IP is reachable; failure_minutes=$failure_minutes fail_count=$FAIL_COUNT"
215
  else
216
    log warning "connectivity check failed for $PING_TARGET; router $ROUTER_IP is not reachable; failure_minutes=$failure_minutes fail_count=$FAIL_COUNT"
217
  fi
218

219
  maybe_recover_router "$now" "$failure_minutes"
220
  maybe_reboot_pve "$now"
221
  save_state
222
}
223

224
main() {
225
  while [[ $# -gt 0 ]]; do
226
    case "$1" in
227
      --dry-run)
228
        DRY_RUN=1
229
        shift
230
        ;;
231
      *)
232
        echo "unknown argument: $1" >&2
233
        exit 2
234
        ;;
235
    esac
236
  done
237

238
  load_config
239
  exec 9>"$LOCK_FILE"
240
  if ! flock -n 9; then
241
    log warning "another watchdog run is active; skipping"
242
    exit 0
243
  fi
244

245
  run_once
246
}
247

248
main "$@"