Linux Troubleshooting

# Show live CPU, memory, load, and top processes.
top

# List the highest CPU processes.
ps aux --sort=-%cpu | head -20

# Show per-process CPU usage every second.
pidstat -u -p REPLACE_PID 1

# Show CPU usage by thread for one process.
ps -T -p REPLACE_PID -o pid,tid,pcpu,pmem,comm

# Attach to a process and see frequent syscalls or blocking calls.
strace -tt -p REPLACE_PID

# Show memory usage, available memory, cache, and swap.
free -h

# Show memory pressure, swap activity, and run queue every second.
vmstat 1

# List the highest memory processes.
ps aux --sort=-%mem | head -20

# Check whether the kernel or cgroup killed a process because of OOM.
dmesg -T | grep -i -E 'oom|killed process'

# Show enabled swap devices and current swap usage.
swapon --show

# Show filesystem space usage.
df -h

# Show inode usage for filesystems.
df -ih

# Find large first-level directories under /var.
du -h --max-depth=1 /var | sort -h

# Find files larger than 1 GiB under /var.
find /var -type f -size +1G -ls

# Find deleted files that are still held open by processes.
lsof +L1

# Show disk utilization, queueing, and latency every second.
iostat -xz 1

# Show processes currently doing disk IO.
iotop -oPa

# Show per-process disk read/write activity every second.
pidstat -d 1

# Show block devices, filesystems, and mount targets.
lsblk -f

# Show listening TCP ports and owning processes.
ss -lntp

# Find which process is using a specific port.
lsof -i :REPLACE_PORT

# Test whether the service works from the same host.
curl -v http://127.0.0.1:REPLACE_PORT/

# Test whether the service works through the target host or IP.
curl -v http://REPLACE_HOST:REPLACE_PORT/

# Show routing table and default gateway.
ip route

# Capture packets for the target port to see whether traffic reaches the host.
tcpdump -i REPLACE_IFACE port REPLACE_PORT

# Show service state, exit code, recent logs, and restart status.
systemctl status REPLACE_SERVICE

# Show recent journal logs for the service.
journalctl -u REPLACE_SERVICE -n 200

# Follow service logs in real time.
journalctl -u REPLACE_SERVICE -f

# Show the effective systemd unit and override files.
systemctl cat REPLACE_SERVICE

# Show environment variables configured in systemd for the service.
systemctl show REPLACE_SERVICE --property=Environment

# Clear systemd failed state after fixing the service.
systemctl reset-failed REPLACE_SERVICE

# Find PID and full command. Replace REPLACE_NAME with the process keyword.
pgrep -af REPLACE_NAME

# Show PID, parent PID, user, terminal, start time, elapsed time, and command.
ps -o pid,ppid,user,tty,stat,lstart,etime,cmd -p REPLACE_PID

# Show the parent chain. This is usually the fastest way to see who started it.
pstree -aps REPLACE_PID

# Show the direct parent process.
ps -fp $(ps -o ppid= -p REPLACE_PID)

# Show executable path, working directory, and command line.
readlink -f /proc/REPLACE_PID/exe
readlink -f /proc/REPLACE_PID/cwd
tr '\0' ' ' < /proc/REPLACE_PID/cmdline; echo

# Show cgroup. This often reveals systemd service, user session, Docker, or Kubernetes.
cat /proc/REPLACE_PID/cgroup

# Ask systemd which unit owns this PID.
systemctl status REPLACE_PID

# If a unit is found, inspect its real startup command and config path.
systemctl cat REPLACE_SERVICE
systemctl show REPLACE_SERVICE --property=MainPID,ExecStart,User,Group,FragmentPath,DropInPaths,Environment

# See recent logs for that service.
journalctl -u REPLACE_SERVICE -n 200

/system.slice/<name>.service:
    system service started by systemd

/user.slice/user-1000.slice/...:
    user session or user-level systemd service

/docker/<container-id> or /system.slice/docker-<id>.scope:
    Docker container process

/kubepods/...:
    Kubernetes pod/container process

/session-<id>.scope:
    login session, ssh session, tmux/screen/manual command possible

grafana  1169  1  0 Jul11 ?  00:02:39 /usr/share/grafana/bin/grafana server --config=/etc/grafana/grafana.ini ...

cat /proc/1169/cgroup

0::/system.slice/grafana-server.service

systemctl status grafana-server.service
systemctl show grafana-server.service -p MainPID -p ExecStart

likely manual foreground:
    parent chain includes sshd -> bash/zsh -> process
    TTY is pts/*
    process dies when terminal/session exits unless protected by tmux/screen/nohup

likely manual background/nohup:
    parent may become PID 1 after shell exits
    no systemd service unit
    cwd may point to user's working directory
    command may include nohup or redirected logs

likely system daemon:
    cgroup points to /system.slice/<service>.service
    systemctl status <pid> finds the unit
    systemctl cat shows ExecStart
    journalctl -u shows logs

likely container:
    cgroup mentions docker/containerd/kubepods
    command may be the container entrypoint
    real startup config is in Docker Compose, Docker inspect, or Kubernetes manifest

# systemd restart policy.
systemctl show <service> --property=Restart,RestartSec,NRestarts

# timers may periodically start jobs.
systemctl list-timers --all

# cron entries.
crontab -l
sudo ls -l /etc/cron.d /etc/cron.hourly /etc/cron.daily /etc/cron.weekly
sudo grep -R '<name>' /etc/cron* 2>/dev/null

# supervisor / pm2.
supervisorctl status
pm2 list

# Docker restart policy.
docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}'
docker inspect <container> --format '{{json .HostConfig.RestartPolicy}}'

crictl ps | grep REPLACE_NAME
kubectl get pod -A -o wide | grep REPLACE_NODE_OR_POD_KEYWORD
kubectl describe pod -n REPLACE_NAMESPACE REPLACE_POD
kubectl get deploy,statefulset,daemonset -n REPLACE_NAMESPACE

# Check whether the current machine's default DNS resolver can resolve the domain.
dig REPLACE_DOMAIN

# Bypass the local resolver and compare the answer with public DNS.
dig @8.8.8.8 REPLACE_DOMAIN

# Check the full HTTP/TLS path: DNS, TCP connect, TLS handshake, cert validation, and HTTP status.
curl -v https://REPLACE_DOMAIN/

# Check TLS handshake, certificate chain, SAN/CN, expiry, and SNI behavior.
openssl s_client -connect REPLACE_HOST:443 -servername REPLACE_DOMAIN

# Check whether system time is obviously wrong.
date

# Check timezone, NTP status, and whether system time is synchronized.
timedatectl