Check Logs for why a System Reboot OR Shutdown in Linux

# 1. When did the last boot happen?
who -b

# 2. List all reboots with timestamps
last reboot | head -20

# 3. Last shutdown/reboot event with adjacent runlevel change
last -x | grep -C1 'shutdown\|reboot' | head -30

# 4. Check previous boot logs (most important on systemd systems)
journalctl -b -1 --no-pager | tail -200

# 5. Check kernel ring buffer from previous boot
journalctl -b -1 -k --no-pager | tail -100

# 6. Did auditd see a clean shutdown or a surprise boot?
ausearch -i -m system_boot,system_shutdown | tail -6
# If two SYSTEM_BOOT lines appear in a row with no SYSTEM_SHUTDOWN between them,
# the system did NOT shut down gracefully — something crashed or lost power.

# Who was logged in just before the reboot?
last | head -30

# Check /var/log/secure (RHEL) or /var/log/auth.log (Debian/Ubuntu) for who ran sudo
ishan-rhel ~]# grep -i 'shutdown\|reboot\|halt\|init 6\|systemctl' /var/log/secure | tail -30

# Check bash history of root and other admin users (not reliable but a starting point)
cat /root/.bash_history | grep -iE 'reboot|shutdown|halt|init 6|systemctl'

# Check audit log for who ran the shutdown command
ausearch -c shutdown --start yesterday --end now -i
ausearch -c reboot   --start yesterday --end now -i
ausearch -c systemctl --start yesterday --end now -i | grep -i reboot

# Check systemd journal for the initiating session
journalctl -b -1 --no-pager | grep -iE 'reboot|shutdown|signal 15|runlevel'

# Check if any automated/scheduled task triggered it (cron, at, systemd timers)
ishan-rhel ~]# cat /var/spool/cron/root
systemctl list-timers --all | grep -i reboot
atq

# Look for ACPI power button events
journalctl -b -1 -k --no-pager | grep -i 'acpi\|power button\|thermal\|critical temp'

# Check current thermal zone readings (post-boot, for trend)
cat /sys/class/thermal/thermal_zone*/temp   # values are in millidegrees C (divide by 1000)

# Check IPMI System Event Log for thermal events (requires ipmitool)
ipmitool sel list | grep -i 'thermal\|temp\|power\|button'

# ACPI events via dmesg
ishan-rhel ~]# dmesg | grep -i 'acpi\|thermal\|power button'

# Check systemd-logind config for power button action
grep -i 'HandlePowerKey\|HandleLidSwitch' /etc/systemd/logind.conf

# Search for OOM events in previous boot
journalctl -b -1 -k --no-pager | grep -i 'out of memory\|oom\|killed process'

# Search all available journal history
journalctl -k --no-pager | grep -i 'out of memory\|oom killer'

# Search /var/log/messages directly (RHEL)
grep -i 'out of memory\|oom\|kill process' /var/log/messages

# Check current OOM panic setting
sysctl vm.panic_on_oom
ishan-rhel ~]# cat /proc/sys/vm/panic_on_oom
# 0 = OOM killer runs (no panic); 1 = panic on OOM; 2 = always panic

# Check current overcommit policy
sysctl vm.overcommit_memory

# Review memory usage around reboot time (if SAR data available)
sar -r -f /var/log/sa/saYYMMDD   # replace with date of reboot

# Check previous boot kernel messages for panic
journalctl -b -1 -k --no-pager | grep -iE 'panic|oops|bug:|general protection|call trace'

# Check auto-reboot on panic setting
sysctl kernel.panic
cat /proc/sys/kernel/panic
# 0 = hang on panic (no reboot); >0 = reboot after N seconds

# Check if panic-on-oops is enabled
sysctl kernel.panic_on_oops

# Check dmesg for oops/panic backtraces
dmesg | grep -A 20 -i 'kernel panic\|BUG:\|Oops:'

# Check if kdump captured a vmcore (see Cause 9 for full kdump section)
ls -lh /var/crash/
less /var/crash/127.0.0.1-2026-06-17-18:10:05/vmcore-dmesg.txt

# Search journal for soft/hard lockup events
journalctl -b -1 -k --no-pager | grep -iE 'soft lockup|hard lockup|nmi watchdog|hung task'

# Check current soft lockup panic setting
sysctl kernel.softlockup_panic
sysctl kernel.hardlockup_panic
sysctl kernel.watchdog_thresh    # default 20 seconds

# Check hung task panic setting
sysctl kernel.hung_task_panic
sysctl kernel.hung_task_timeout_secs   # default 120 seconds

# Check if watchdog is enabled
sysctl kernel.watchdog

# Pacemaker cluster and fencing logs
journalctl -b -1 --no-pager -u pacemaker | grep -iE 'fenc|stonith|reboot|shot'
journalctl -b -1 --no-pager -u corosync  | tail -50

# Dedicated fencing log (if configured)
grep -iE 'fenc|stonith|reboot' /var/log/pacemaker/pacemaker.log | tail -50

# Check cluster history for fencing events (pcs)
pcs status
pcs stonith history

# Check Corosync ring/quorum at time of incident
grep -i 'quorum\|ring\|lost\|partition' /var/log/cluster/corosync.log | tail -30

# crm_report captures everything for a time window
crm_report -f "YYYY-MM-DD HH:MM:SS" -t "YYYY-MM-DD HH:MM:SS" /tmp/crm-report

# Check IPMI SEL for watchdog timer expiry events
ipmitool sel list | grep -i 'watchdog\|timer\|reset\|power'

# Get full IPMI event log with decoded descriptions
ipmitool sel elist

# Check if ipmi_watchdog module is loaded and its settings
lsmod | grep ipmi_watchdog
cat /sys/module/ipmi_watchdog/parameters/action       # reset, power_cycle, power_off, none
cat /sys/module/ipmi_watchdog/parameters/timeout      # countdown in seconds
cat /sys/module/ipmi_watchdog/parameters/pretimeout   # pre-NMI seconds

# Check watchdog daemon status (if using watchdog package)
systemctl status watchdog

# Review BMC System Event Log entries around reboot time
ipmitool sel time get   # confirm BMC clock vs system clock
ipmitool sdr type 'Watchdog'

# Check if SysRq crash was triggered
journalctl -b -1 -k --no-pager | grep -i 'sysrq\|trigger a crash'
grep -i sysrq /var/log/messages

# Check current SysRq enabled bitmap
sysctl kernel.sysrq
cat /proc/sys/kernel/sysrq
# 0=disabled, 1=all, 176=safe subset (RHEL default in some versions)

# Who could have done it? Check audit log
ausearch -f /proc/sysrq-trigger --start yesterday --end now -i

# Check /etc/sysctl.conf and /etc/sysctl.d/ for intentional configuration
grep -r sysrq /etc/sysctl.conf /etc/sysctl.d/

# Is kexec-tools installed?
rpm -q kexec-tools

# Is kdump service enabled and active?
systemctl status kdump

# How much crash memory is reserved?
cat /proc/cmdline | grep -o 'crashkernel=[^ ]*'
# or
grep crashkernel /etc/default/grub

# Where will vmcore be saved?
grep -v '^#' /etc/kdump.conf | grep -v '^$'

# List crash dumps captured
ls -lh /var/crash/

# Identify which vmcore belongs to which crash time
ls -lh /var/crash/*/vmcore

# Install crash and kernel debug symbols (RHEL)
dnf install crash
dnf install kernel-debuginfo-$(uname -r)

# Open the vmcore with crash utility
crash /usr/lib/debug/lib/modules/$(uname -r)/vmlinux /var/crash/<timestamp>/vmcore

# Inside crash utility — most useful commands:
# bt         — backtrace of the crashing CPU at time of panic
# log        — kernel message buffer (dmesg at crash time)
# ps         — process list at crash time
# vm         — virtual memory info
# sys        — system info (uptime, kernel version, panic string)
# q          — quit

# Enable and start kdump
systemctl enable --now kdump

# Verify crash kernel is reserved
grep -i crashkernel /proc/cmdline

# If not set, add crashkernel= to GRUB and reboot
# RHEL 8/9 (BIOS):
grubby --update-kernel=ALL --args="crashkernel=auto"
# RHEL 8/9 (UEFI):
grubby --update-kernel=ALL --args="crashkernel=auto"

# Test kdump is functional (THIS WILL CRASH THE SYSTEM — test env only)
echo c > /proc/sysrq-trigger

# Check MCE log via mcelog (older, RHEL 6/7)
mcelog --client   # if mcelog daemon is running
cat /var/log/mcelog

# RHEL 8/9+: use rasdaemon instead of mcelog
systemctl status rasdaemon
ras-mc-ctl --summary
ras-mc-ctl --errors

# Check kernel MCE messages
journalctl -b -1 -k --no-pager | grep -iE 'mce|machine check|edac|ecc|uncorrect'
dmesg | grep -iE 'mce|machine check|edac|ecc'

# Check IPMI SEL for memory/hardware errors
ipmitool sel elist | grep -iE 'mem|ecc|correctable|uncorrectable|dimm'

# Check EDAC (Error Detection And Correction) subsystem
ls /sys/devices/system/edac/mc/
cat /sys/devices/system/edac/mc/mc*/ce_count    # correctable errors
cat /sys/devices/system/edac/mc/mc*/ue_count    # uncorrectable errors (critical)

# Find D-state processes right now (if system is still responsive)
ps aux | awk '$8 == "D"'

# Check for hung task kernel messages in previous boot
journalctl -b -1 -k --no-pager | grep -iE 'blocked for more than|hung task|state:D'
grep 'blocked for more than' /var/log/messages

# Check hung_task settings
sysctl kernel.hung_task_timeout_secs    # 0 = disabled
sysctl kernel.hung_task_panic           # 1 = panic when hung task detected

# Check NFS mounts for stale/hanging connections
mount | grep nfs
nfsstat -m
cat /proc/mounts | grep nfs

# Check storage/multipath for path failures
multipath -ll
dmsetup status

# If the journal simply ends with no shutdown sequence → power cut
journalctl -b -1 --no-pager | tail -30
# A normal shutdown ends with lines like "Reached target Power-Off"
# An abrupt power loss: the journal ends mid-stream with no shutdown messages

# Check auditd for two consecutive boots with no shutdown between them
ausearch -i -m system_boot,system_shutdown | tail -8

# Check apcupsd logs (if installed)
cat /var/log/apcupsd.events

# Check NUT (Network UPS Tools) logs
journalctl -b -1 --no-pager -u nut-monitor
journalctl -b -1 --no-pager -u upsmon

# Check IPMI SEL for power loss events
ipmitool sel elist | grep -iE 'power\|ac lost\|battery'