diff --git a/Debian网络异常和日志查看命令记录.md b/Debian网络异常和日志查看命令记录.md new file mode 100644 index 0000000..3b00c00 --- /dev/null +++ b/Debian网络异常和日志查看命令记录.md @@ -0,0 +1,222 @@ +# Debian网络异常和日志查看命令记录 +## 1. 手动协商命令 +```bash +ethtool -r lan2 +``` + +## 2. WatchDog日志查看。 + +看最近运行日志 +```bash +journalctl -u lan2-link-watchdog.service -n 100 --no-pager +``` + +实时看: +```bash +journalctl -u lan2-link-watchdog.service -f +``` +只看有没有触发过重新协商 + +这个最有用: +```bash +journalctl -u lan2-link-watchdog.service --no-pager | grep -E 'BAD|ACTION|VERIFY|cooldown' +``` +如果没输出,说明从启动到现在没有触发过异常处理。 + +如果触发过,会看到类似: +```bash +BAD round 1/2: received 92/100, loss=8 +BAD round 2/2: received 90/100, loss=10 +ACTION: renegotiate lan2 because link quality is bad +VERIFY after renegotiate: received 100/100 +cooldown 300s +``` +其中关键是这行: +```bash +ACTION: renegotiate lan2 because link quality is bad +``` +看到它就说明 watchdog 执行过: +```bash +ethtool -r lan2 +``` +看今天触发过几次 +```bash +journalctl -u lan2-link-watchdog.service --since today --no-pager | grep -c 'ACTION: renegotiate' +``` +输出 0 就是今天没触发。 + +看当前这次开机以来有没有触发 +```bash +journalctl -b -u lan2-link-watchdog.service --no-pager | grep -E 'BAD|ACTION|VERIFY|cooldown' +``` +看当前状态 +```bash +systemctl status lan2-link-watchdog.service --no-pager +``` +另外,因为脚本里有这句: +```bash +logger -t lan2-watchdog "$*" +``` +所以你也可以按 tag 查: +```bash +journalctl -t lan2-watchdog --no-pager +``` +不过平时用这个就够了: +```bash +journalctl -u lan2-link-watchdog.service --no-pager | grep -E 'BAD|ACTION|VERIFY|cooldown' +``` +如果以后重启后想保留更久的日志,可以确认一下 journald 是否持久化: +```bash +journalctl --list-boots +``` +如果能看到多次 boot 记录,说明日志是保留的。否则以后我再帮你改成单独写 /var/log/lan2-link-watchdog.log 也行。 + +## 3. 创建 lan3 watchdog 脚本 +```bash +sudo tee /usr/local/sbin/lan3-link-watchdog.sh >/dev/null <<'EOF' +#!/bin/bash + +IFACE="lan3" +CHECK_IFACE="lan3" +GW="192.168.3.1" + +COUNT=100 +MIN_RECV=99 # RECV < 99 才算异常;也就是 98/100 或更低 +BAD_ROUNDS_LIMIT=2 # 连续 2 轮异常才执行 ethtool -r +SLEEP_OK=60 # 正常时,每轮结束后休息 60 秒 +COOLDOWN=300 # 重协商后冷却 300 秒 +ARPING_TIMEOUT=130 # arping 100 次大约 100 秒,给 130 秒超时 + +BAD_ROUNDS=0 + +log() { + logger -t lan3-watchdog "$*" + echo "$(date '+%F %T') $*" +} + +ARPING_BIN="$(command -v arping || true)" +ETHTOOL_BIN="$(command -v ethtool || true)" + +if [ -z "$ARPING_BIN" ]; then + log "ERROR: arping not found" + exit 1 +fi + +if [ -z "$ETHTOOL_BIN" ]; then + log "ERROR: ethtool not found" + exit 1 +fi + +log "started: IFACE=$IFACE CHECK_IFACE=$CHECK_IFACE GW=$GW COUNT=$COUNT MIN_RECV=$MIN_RECV BAD_ROUNDS_LIMIT=$BAD_ROUNDS_LIMIT" + +while true; do + if [ ! -d "/sys/class/net/$IFACE" ]; then + log "WARN: interface $IFACE not found, sleep ${SLEEP_OK}s" + sleep "$SLEEP_OK" + continue + fi + + if [ ! -d "/sys/class/net/$CHECK_IFACE" ]; then + log "WARN: check interface $CHECK_IFACE not found, sleep ${SLEEP_OK}s" + sleep "$SLEEP_OK" + continue + fi + + OUT=$(LANG=C timeout "$ARPING_TIMEOUT" "$ARPING_BIN" -I "$CHECK_IFACE" -c "$COUNT" "$GW" 2>&1) + + SENT=$(echo "$OUT" | awk '/^Sent / {print $2}') + RECV=$(echo "$OUT" | awk '/^Received / {print $2}') + + if [ -z "$RECV" ]; then + BAD_ROUNDS=$((BAD_ROUNDS + 1)) + log "WARN: could not parse arping result, bad round $BAD_ROUNDS/$BAD_ROUNDS_LIMIT" + log "$OUT" + + if [ "$BAD_ROUNDS" -ge "$BAD_ROUNDS_LIMIT" ]; then + log "ACTION: renegotiate $IFACE because arping result could not be parsed repeatedly" + "$ETHTOOL_BIN" -r "$IFACE" + + BAD_ROUNDS=0 + log "cooldown ${COOLDOWN}s" + sleep "$COOLDOWN" + else + sleep "$SLEEP_OK" + fi + + continue + fi + + LOSS=$((COUNT - RECV)) + + if [ "$RECV" -gt "$COUNT" ]; then + log "OK with extra replies: received $RECV/$COUNT, likely another arping is running" + BAD_ROUNDS=0 + sleep "$SLEEP_OK" + continue + fi + + if [ "$RECV" -lt "$MIN_RECV" ]; then + BAD_ROUNDS=$((BAD_ROUNDS + 1)) + log "BAD round $BAD_ROUNDS/$BAD_ROUNDS_LIMIT: received $RECV/$COUNT, loss=$LOSS" + + if [ "$BAD_ROUNDS" -ge "$BAD_ROUNDS_LIMIT" ]; then + log "ACTION: renegotiate $IFACE because link quality is bad" + "$ETHTOOL_BIN" -r "$IFACE" + + sleep 5 + + VERIFY=$(LANG=C timeout "$ARPING_TIMEOUT" "$ARPING_BIN" -I "$CHECK_IFACE" -c "$COUNT" "$GW" 2>&1) + VERIFY_RECV=$(echo "$VERIFY" | awk '/^Received / {print $2}') + + log "VERIFY after renegotiate: received ${VERIFY_RECV:-unknown}/$COUNT" + + BAD_ROUNDS=0 + log "cooldown ${COOLDOWN}s" + sleep "$COOLDOWN" + else + sleep "$SLEEP_OK" + fi + else + if [ "$BAD_ROUNDS" -ne 0 ]; then + log "RECOVERED without action: received $RECV/$COUNT" + fi + + log "OK: received $RECV/$COUNT" + BAD_ROUNDS=0 + sleep "$SLEEP_OK" + fi +done +EOF +``` +给脚本赋权限 +```bash +sudo chmod +x /usr/local/sbin/lan3-link-watchdog.sh +``` + +## 4. 创建 lan3 systemd 服务 +```bash +sudo tee /etc/systemd/system/lan3-link-watchdog.service >/dev/null <<'EOF' +[Unit] +Description=Watch lan3 link quality and renegotiate on packet loss +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/local/sbin/lan3-link-watchdog.sh +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +EOF +``` +然后加载一下: +```bash +sudo systemctl daemon-reload +``` +确认 lan3 通了之后,再启用: +```bash +sudo systemctl enable lan3-link-watchdog.service +sudo systemctl start lan3-link-watchdog.service +```