Linux Tutorial (11): Performance Tuning dan Troubleshooting Production

# Load average vs CPU cores
uptime
nproc
 
# Per-CPU usage
mpstat -P ALL 1 5
 
# Top CPU consumers
ps aux --sort=-%cpu | head -10
pidstat 1 5
 
# Context switches & interrupts
vmstat 1 5
# cs = context switches/sec, in = interrupts/sec

# I/O wait visible in top (%wa) or mpstat (%iowait)
top    # lihat %wa
iostat -x 1 5   # await, %util per device

free -h
cat /proc/meminfo | head -20
 
# Detail per proses
ps aux --sort=-%mem | head -10
pmap -x $(pidof nginx) | tail -1
 
# Slab & cache
slabtop

total   used   free   buff/cache   available
              ↑                        ↑
         bukan masalah            yang benar-benar bisa dipakai

# OOM killer log
dmesg | grep -i "out of memory"
journalctl -k | grep -i "oom"
grep -i "oom" /var/log/syslog

# Per-device stats
iostat -xmdz 1 5
 
# Kolom penting:
# %util  → device busy (near 100% = saturated)
# await  → avg wait time ms (high = slow disk or queue)
# r/s w/s → read/write ops per second
# rkB/s wkB/s → throughput
 
# Per-process I/O
sudo iotop -oP    # hanya proses dengan I/O
 
# Identify heavy files being accessed
sudo lsof | awk '{print $9}' | sort | uniq -c | sort -rn | head -20

# Bandwidth per interface
sar -n DEV 1 5
ip -s link show eth0
 
# Connection stats
ss -s
ss -tn state time-wait | wc -l
 
# Packet drops/errors
netstat -i
ip -s link
 
# Capture traffic
sudo tcpdump -i eth0 port 443 -c 100
sudo tcpdump -i any host 10.0.0.5 -w capture.pcap

# Trace system calls
strace -p $(pidof nginx) -f -e trace=network
 
# Debug startup hang
strace -f -o /tmp/trace.log ./slow-start-app
grep -E "EACCES|ENOENT|ECONNREFUSED" /tmp/trace.log
 
# Count syscalls
strace -c -p $(pidof mysql)

# Semua open files proses
lsof -p 1234
 
# Port yang dibuka
lsof -i :8080
lsof -i TCP:443 -s TCP:LISTEN
 
# File deleted tapi masih held (space not freed!)
lsof +L1
 
# Fix deleted-but-open file:
# Restart proses, atau truncate via /proc/PID/fd/FD

sudo apt install linux-tools-common linux-tools-$(uname -r)
 
# CPU flamegraph data
sudo perf record -F 99 -p $(pidof app) -g -- sleep 30
sudo perf report
 
# Quick stat
sudo perf stat -p $(pidof app) sleep 10

# Step 1: Quick health check (30 detik)
uptime && free -h && df -h && ss -s
 
# Step 2: Identify bottleneck
vmstat 1 5          # CPU vs I/O wait
iostat -x 1 5       # Disk saturation
sar -n DEV 1 5      # Network
 
# Step 3: Find culprit process
ps aux --sort=-%cpu | head -5
ps aux --sort=-%mem | head -5
sudo iotop -oP
 
# Step 4: Deep dive
strace -p CULPRIT_PID -c
lsof -p CULPRIT_PID

df -h
sudo du -xh /var --max-depth=2 | sort -rh | head -10
sudo lsof +L1                    # deleted files still open
sudo journalctl --disk-usage
sudo journalctl --vacuum-size=500M

ss -tlnp | grep :8080            # listening?
sudo ufw status                  # firewall?
curl -v localhost:8080           # local works?
curl -v server-ip:8080           # remote works?
sudo tcpdump -i any port 8080 -c 10   # packets arriving?

ps aux | awk '$8 ~ /D/ {print}'   # uninterruptible sleep (I/O)
ps aux | awk '$8 ~ /Z/ {print}'   # zombie
cat /proc/PID/stack               # kernel stack trace
strace -p PID                     # stuck on what syscall?

# /etc/sysctl.d/99-performance.conf
 
# Network tuning (high-traffic web server)
net.core.somaxconn = 65535
net.ipv4.tcp_max_syn_backlog = 65535
net.core.netdev_max_backlog = 65535
net.ipv4.tcp_fin_timeout = 15
net.ipv4.tcp_tw_reuse = 1
 
# Memory
vm.swappiness = 10               # prefer RAM over swap
vm.dirty_ratio = 15
vm.dirty_background_ratio = 5
 
# File descriptors
fs.file-max = 2097152

# Per-service limits: /etc/systemd/system/nginx.service.d/limits.conf
[Service]
LimitNOFILE=65535
LimitNPROC=65535

# Per-user limits: /etc/security/limits.d/app.conf
appuser soft nofile 65535
appuser hard nofile 65535

Metrics collection  →  Prometheus node_exporter / Telegraf
Visualization       →  Grafana
Alerting            →  Alertmanager / PagerDuty
Log aggregation     →  Loki / ELK Stack
Tracing             →  Jaeger / OpenTelemetry

# Quick local monitoring
sudo apt install sysstat
sar -u 1 5     # CPU history
sar -r 1 5     # Memory history
sar -b 1 5     # I/O history