add Debian网络监控工具
This commit is contained in:
@@ -0,0 +1,222 @@
|
|||||||
|
# Debian网络异常和日志查看命令记录
|
||||||
|
## 1. 手动协商命令
|
||||||
|
```bash
|
||||||
|
ethtool -r lan2
|
||||||
|
```
|
||||||
|
|
||||||
|
## 2. WatchDog日志查看。
|
||||||
|
|
||||||
|
看最近运行日志
|
||||||
|
```bash
|
||||||
|
journalctl -u lan2-link-watchdog.service -n 100 --no-pager
|
||||||
|
```
|
||||||
|
|
||||||
|
实时看：
|
||||||
|
```bash
|
||||||
|
journalctl -u lan2-link-watchdog.service -f
|
||||||
|
```
|
||||||
|
只看有没有触发过重新协商
|
||||||
|
|
||||||
|
这个最有用：
|
||||||
|
```bash
|
||||||
|
journalctl -u lan2-link-watchdog.service --no-pager | grep -E 'BAD|ACTION|VERIFY|cooldown'
|
||||||
|
```
|
||||||
|
如果没输出，说明从启动到现在没有触发过异常处理。
|
||||||
|
|
||||||
|
如果触发过，会看到类似：
|
||||||
|
```bash
|
||||||
|
BAD round 1/2: received 92/100, loss=8
|
||||||
|
BAD round 2/2: received 90/100, loss=10
|
||||||
|
ACTION: renegotiate lan2 because link quality is bad
|
||||||
|
VERIFY after renegotiate: received 100/100
|
||||||
|
cooldown 300s
|
||||||
|
```
|
||||||
|
其中关键是这行：
|
||||||
|
```bash
|
||||||
|
ACTION: renegotiate lan2 because link quality is bad
|
||||||
|
```
|
||||||
|
看到它就说明 watchdog 执行过：
|
||||||
|
```bash
|
||||||
|
ethtool -r lan2
|
||||||
|
```
|
||||||
|
看今天触发过几次
|
||||||
|
```bash
|
||||||
|
journalctl -u lan2-link-watchdog.service --since today --no-pager | grep -c 'ACTION: renegotiate'
|
||||||
|
```
|
||||||
|
输出 0 就是今天没触发。
|
||||||
|
|
||||||
|
看当前这次开机以来有没有触发
|
||||||
|
```bash
|
||||||
|
journalctl -b -u lan2-link-watchdog.service --no-pager | grep -E 'BAD|ACTION|VERIFY|cooldown'
|
||||||
|
```
|
||||||
|
看当前状态
|
||||||
|
```bash
|
||||||
|
systemctl status lan2-link-watchdog.service --no-pager
|
||||||
|
```
|
||||||
|
另外，因为脚本里有这句：
|
||||||
|
```bash
|
||||||
|
logger -t lan2-watchdog "$*"
|
||||||
|
```
|
||||||
|
所以你也可以按 tag 查：
|
||||||
|
```bash
|
||||||
|
journalctl -t lan2-watchdog --no-pager
|
||||||
|
```
|
||||||
|
不过平时用这个就够了：
|
||||||
|
```bash
|
||||||
|
journalctl -u lan2-link-watchdog.service --no-pager | grep -E 'BAD|ACTION|VERIFY|cooldown'
|
||||||
|
```
|
||||||
|
如果以后重启后想保留更久的日志，可以确认一下 journald 是否持久化：
|
||||||
|
```bash
|
||||||
|
journalctl --list-boots
|
||||||
|
```
|
||||||
|
如果能看到多次 boot 记录，说明日志是保留的。否则以后我再帮你改成单独写 /var/log/lan2-link-watchdog.log 也行。
|
||||||
|
|
||||||
|
## 3. 创建 lan3 watchdog 脚本
|
||||||
|
```bash
|
||||||
|
sudo tee /usr/local/sbin/lan3-link-watchdog.sh >/dev/null <<'EOF'
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
IFACE="lan3"
|
||||||
|
CHECK_IFACE="lan3"
|
||||||
|
GW="192.168.3.1"
|
||||||
|
|
||||||
|
COUNT=100
|
||||||
|
MIN_RECV=99 # RECV < 99 才算异常；也就是 98/100 或更低
|
||||||
|
BAD_ROUNDS_LIMIT=2 # 连续 2 轮异常才执行 ethtool -r
|
||||||
|
SLEEP_OK=60 # 正常时，每轮结束后休息 60 秒
|
||||||
|
COOLDOWN=300 # 重协商后冷却 300 秒
|
||||||
|
ARPING_TIMEOUT=130 # arping 100 次大约 100 秒，给 130 秒超时
|
||||||
|
|
||||||
|
BAD_ROUNDS=0
|
||||||
|
|
||||||
|
log() {
|
||||||
|
logger -t lan3-watchdog "$*"
|
||||||
|
echo "$(date '+%F %T') $*"
|
||||||
|
}
|
||||||
|
|
||||||
|
ARPING_BIN="$(command -v arping || true)"
|
||||||
|
ETHTOOL_BIN="$(command -v ethtool || true)"
|
||||||
|
|
||||||
|
if [ -z "$ARPING_BIN" ]; then
|
||||||
|
log "ERROR: arping not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "$ETHTOOL_BIN" ]; then
|
||||||
|
log "ERROR: ethtool not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "started: IFACE=$IFACE CHECK_IFACE=$CHECK_IFACE GW=$GW COUNT=$COUNT MIN_RECV=$MIN_RECV BAD_ROUNDS_LIMIT=$BAD_ROUNDS_LIMIT"
|
||||||
|
|
||||||
|
while true; do
|
||||||
|
if [ ! -d "/sys/class/net/$IFACE" ]; then
|
||||||
|
log "WARN: interface $IFACE not found, sleep ${SLEEP_OK}s"
|
||||||
|
sleep "$SLEEP_OK"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -d "/sys/class/net/$CHECK_IFACE" ]; then
|
||||||
|
log "WARN: check interface $CHECK_IFACE not found, sleep ${SLEEP_OK}s"
|
||||||
|
sleep "$SLEEP_OK"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
OUT=$(LANG=C timeout "$ARPING_TIMEOUT" "$ARPING_BIN" -I "$CHECK_IFACE" -c "$COUNT" "$GW" 2>&1)
|
||||||
|
|
||||||
|
SENT=$(echo "$OUT" | awk '/^Sent / {print $2}')
|
||||||
|
RECV=$(echo "$OUT" | awk '/^Received / {print $2}')
|
||||||
|
|
||||||
|
if [ -z "$RECV" ]; then
|
||||||
|
BAD_ROUNDS=$((BAD_ROUNDS + 1))
|
||||||
|
log "WARN: could not parse arping result, bad round $BAD_ROUNDS/$BAD_ROUNDS_LIMIT"
|
||||||
|
log "$OUT"
|
||||||
|
|
||||||
|
if [ "$BAD_ROUNDS" -ge "$BAD_ROUNDS_LIMIT" ]; then
|
||||||
|
log "ACTION: renegotiate $IFACE because arping result could not be parsed repeatedly"
|
||||||
|
"$ETHTOOL_BIN" -r "$IFACE"
|
||||||
|
|
||||||
|
BAD_ROUNDS=0
|
||||||
|
log "cooldown ${COOLDOWN}s"
|
||||||
|
sleep "$COOLDOWN"
|
||||||
|
else
|
||||||
|
sleep "$SLEEP_OK"
|
||||||
|
fi
|
||||||
|
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
LOSS=$((COUNT - RECV))
|
||||||
|
|
||||||
|
if [ "$RECV" -gt "$COUNT" ]; then
|
||||||
|
log "OK with extra replies: received $RECV/$COUNT, likely another arping is running"
|
||||||
|
BAD_ROUNDS=0
|
||||||
|
sleep "$SLEEP_OK"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$RECV" -lt "$MIN_RECV" ]; then
|
||||||
|
BAD_ROUNDS=$((BAD_ROUNDS + 1))
|
||||||
|
log "BAD round $BAD_ROUNDS/$BAD_ROUNDS_LIMIT: received $RECV/$COUNT, loss=$LOSS"
|
||||||
|
|
||||||
|
if [ "$BAD_ROUNDS" -ge "$BAD_ROUNDS_LIMIT" ]; then
|
||||||
|
log "ACTION: renegotiate $IFACE because link quality is bad"
|
||||||
|
"$ETHTOOL_BIN" -r "$IFACE"
|
||||||
|
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
VERIFY=$(LANG=C timeout "$ARPING_TIMEOUT" "$ARPING_BIN" -I "$CHECK_IFACE" -c "$COUNT" "$GW" 2>&1)
|
||||||
|
VERIFY_RECV=$(echo "$VERIFY" | awk '/^Received / {print $2}')
|
||||||
|
|
||||||
|
log "VERIFY after renegotiate: received ${VERIFY_RECV:-unknown}/$COUNT"
|
||||||
|
|
||||||
|
BAD_ROUNDS=0
|
||||||
|
log "cooldown ${COOLDOWN}s"
|
||||||
|
sleep "$COOLDOWN"
|
||||||
|
else
|
||||||
|
sleep "$SLEEP_OK"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if [ "$BAD_ROUNDS" -ne 0 ]; then
|
||||||
|
log "RECOVERED without action: received $RECV/$COUNT"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "OK: received $RECV/$COUNT"
|
||||||
|
BAD_ROUNDS=0
|
||||||
|
sleep "$SLEEP_OK"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
给脚本赋权限
|
||||||
|
```bash
|
||||||
|
sudo chmod +x /usr/local/sbin/lan3-link-watchdog.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. 创建 lan3 systemd 服务
|
||||||
|
```bash
|
||||||
|
sudo tee /etc/systemd/system/lan3-link-watchdog.service >/dev/null <<'EOF'
|
||||||
|
[Unit]
|
||||||
|
Description=Watch lan3 link quality and renegotiate on packet loss
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
ExecStart=/usr/local/sbin/lan3-link-watchdog.sh
|
||||||
|
Restart=always
|
||||||
|
RestartSec=10
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
然后加载一下：
|
||||||
|
```bash
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
```
|
||||||
|
确认 lan3 通了之后，再启用：
|
||||||
|
```bash
|
||||||
|
sudo systemctl enable lan3-link-watchdog.service
|
||||||
|
sudo systemctl start lan3-link-watchdog.service
|
||||||
|
```
|
||||||
Reference in New Issue
Block a user