Linux — Practical

Linux — Practical (commands & one-liners)

Top resource consumers

# top by mem
ps -eo pid,user,%mem,rss,vsz,cmd --sort=-rss | head

# top by cpu
ps -eo pid,user,%cpu,cmd --sort=-%cpu | head

# group by user
ps -eo user,rss --no-headers | awk '{a[$1]+=$2} END {for(u in a) print u, a[u]/1024 " MB"}'

File system inspection

df -h                                  # disk usage
df -i                                  # inodes
du -sh /var/log/*                      # by dir
du -ah . | sort -rh | head -20         # biggest files

# deleted but open files (still hold disk)
sudo lsof | grep deleted | head

# what's mounted, types
mount | column -t
findmnt

Network triage

# listening sockets
sudo ss -tnlp

# connections to port 5432
ss -tn 'sport = :5432' | head

# latency / route
mtr --report --report-cycles=10 host
traceroute -T -p 443 host

# DNS resolution
dig +short example.com
dig +trace example.com

# packet capture
sudo tcpdump -nn -i any -s 0 -w cap.pcap port 8080

# raw socket count by state
ss -tan | awk '{print $1}' | sort | uniq -c

Process inspection

# what's it doing?
strace -p PID -e trace=network 2>&1 | head
sudo cat /proc/PID/status
cat /proc/PID/wchan; cat /proc/PID/stack
ls -la /proc/PID/fd | head
sudo lsof -p PID
sudo gcore PID                          # core dump

# tree of processes
pstree -p

# threads
ps -L -p PID
top -H -p PID                           # per-thread CPU

Memory deep

# meminfo
cat /proc/meminfo | head -20

# swap usage by process
for f in /proc/[0-9]*/status; do awk '/Name|VmSwap/{printf "%s ", $2}' "$f"; echo; done | sort -k2 -n -r | head

# huge pages
cat /proc/meminfo | grep -i huge

# slab
sudo slabtop -o | head

Logs

# system journal
journalctl -u nginx -f --since '10 min ago' --no-pager
journalctl -p err --since today
journalctl -k                           # kernel only
journalctl --vacuum-size=500M            # trim

# rotate logs
sudo logrotate -d /etc/logrotate.d/nginx

# greps
zgrep -h ERROR /var/log/syslog.* | tail

Performance one-shots (USE method)

# CPU
mpstat -P ALL 1 5

# memory
sar -r 1 5

# disk
iostat -x 1 5

# net
sar -n DEV 1 5

# all-in-one
dstat -tcndmgyl 1 10

eBPF (bcc / bpftrace)

# top syscalls per process
sudo bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); } interval:s:5 { print(@); clear(@); }'

# slow file reads
sudo /usr/share/bcc/tools/biolatency
sudo /usr/share/bcc/tools/biotop
sudo /usr/share/bcc/tools/tcpconnect
sudo /usr/share/bcc/tools/opensnoop

Brendan Gregg’s bcc tools and bpftrace give visibility without restarts or instrumentation.

Useful flags / kernel tunings

# ephemeral port range
sysctl net.ipv4.ip_local_port_range

# raise file descriptors per process
ulimit -n 1048576

# keep TCP keep-alive close-ish
sysctl -w net.ipv4.tcp_keepalive_time=60
sysctl -w net.ipv4.tcp_keepalive_intvl=10
sysctl -w net.ipv4.tcp_keepalive_probes=6

# allow more connections waiting
sysctl -w net.core.somaxconn=65535
sysctl -w net.ipv4.tcp_max_syn_backlog=65535

# avoid TIME_WAIT exhaustion
sysctl -w net.ipv4.tcp_tw_reuse=1

Persist in /etc/sysctl.d/99-tuning.conf.

Useful one-liners

# count reqs by HTTP status from access log
awk '{print $9}' access.log | sort | uniq -c | sort -rn

# top URLs
awk '{print $7}' access.log | sort | uniq -c | sort -rn | head

# replace string across files
grep -rl 'old' . | xargs sed -i 's/old/new/g'

# files modified in last hour
find /var -type f -mmin -60

# process tree of biggest mem hog
pstree -ps $(ps -eo pid,rss --sort=-rss --no-headers | head -1 | awk '{print $1}')

# busy disk
sudo iotop -o -b -n 5 -d 1

SSH config power

Host bastion
  HostName bastion.example.com
  User ubuntu
  IdentityFile ~/.ssh/id_ed25519

Host *.internal
  ProxyJump bastion
  User ubuntu
  ServerAliveInterval 60

Host db1.internal
  LocalForward 5432 localhost:5432

ssh db1.internal           # tunnels through bastion
psql -h localhost -p 5432

systemd-run for ad-hoc

systemd-run --user --scope -- nice -n 19 myjob

# resource-limited transient unit
systemd-run --uid=app --slice=mybatch.slice -p MemoryMax=512M -p CPUQuota=50% bash batch.sh

Container debugging on host

# find pid of container process
docker top <id> | awk 'NR>1 {print $2}'

# nsenter into pid for debugging
sudo nsenter -t PID -n ss -tnp     # inspect namespace from host