Linux Tutorial (8): Text Processing: grep, sed, dan awk

# Output satu command = input command berikutnya
cat /var/log/nginx/access.log | grep "POST" | awk '{print $1}' | sort | uniq -c | sort -rn | head -10

# Basic search
grep "error" /var/log/syslog
grep -i "error" app.log              # case-insensitive
grep -r "password" /etc/             # recursive
grep -n "listen" nginx.conf          # show line number
grep -c "404" access.log             # count matches
grep -v "127.0.0.1" access.log       # invert (exclude)
grep -A 3 "Exception" app.log          # 3 lines after match
grep -B 2 -A 5 "FATAL" app.log         # context
 
# Extended regex (-E = egrep)
grep -E "error|warn|fatal" syslog
grep -E "^[0-9]{4}-[0-9]{2}-[0-9]{2}" log.txt
 
# Fixed string (no regex)
grep -F "price=$100" data.txt
 
# Only filenames
grep -rl "TODO" src/

# Substitute (s/pattern/replacement/flags)
sed 's/error/ERROR/g' log.txt         # global replace
sed 's/^/PREFIX: /' file.txt          # prepend every line
sed 's/[0-9]\+/XXX/g' file.txt        # replace numbers
 
# In-place edit (backup dulu!)
sed -i.bak 's/debug/info/g' config.ini
sed -i 's/port=8080/port=3000/' .env
 
# Delete lines
sed '/^#/d' config.txt                # delete comments
sed '/^$/d' file.txt                  # delete blank lines
sed '1,10d' file.txt                  # delete lines 1-10
 
# Print range
sed -n '10,20p' large-file.log        # print lines 10-20
sed -n '/ERROR/p' app.log             # print lines matching pattern
 
# Multiple commands
sed -e 's/foo/bar/g' -e '/^$/d' file.txt

# Uncomment line
sed -i 's/#PermitRootLogin no/PermitRootLogin no/' /etc/ssh/sshd_config
 
# Insert line after pattern
sed -i '/\[mysqld\]/a character-set-server=utf8mb4' /etc/mysql/my.cnf

# Print kolom
awk '{print $1}' access.log           # kolom 1 (IP)
awk '{print $NF}' file.txt            # kolom terakhir
awk '{print $(NF-1)}' file.txt        # kolom kedua dari belakang
 
# Custom separator
awk -F: '{print $1, $3}' /etc/passwd  # user dan UID
awk -F',' '{print $2}' data.csv       # CSV kolom 2
 
# Conditional
awk '$3 > 100 {print $1, $3}' data.txt
awk '/ERROR/ {print $0}' app.log
awk 'NR >= 10 && NR <= 20' file.txt   # line number range
 
# Aggregation
awk '{sum += $3} END {print sum}' numbers.txt
awk '{count[$1]++} END {for (ip in count) print count[ip], ip}' access.log | sort -rn

# Parse nginx access log: top 10 IP
awk '{
  ip = $1
  count[ip]++
  bytes[ip] += $10
}
END {
  for (ip in count)
    printf "%-15s %6d requests  %10d bytes\n", ip, count[ip], bytes[ip]
}' /var/log/nginx/access.log | sort -k2 -rn | head -10

# Report disk usage per user
awk -F: '{print $1, $3, $6}' /etc/passwd | while read user uid home; do
  if [[ -d "$home" ]]; then
    size=$(du -sh "$home" 2>/dev/null | cut -f1)
    echo "$user (uid=$uid): $size"
  fi
done

# Error rate per jam dari syslog
grep "error" /var/log/syslog \
  | awk '{print $1, $2, $3}' \
  | cut -d: -f1 \
  | sort \
  | uniq -c \
  | sort -rn
 
# HTTP status code distribution
awk '{print $9}' /var/log/nginx/access.log \
  | sort \
  | uniq -c \
  | sort -rn
 
# Slow requests (> 1 detik)
awk '$NF > 1.0 {print $7, $NF "s"}' /var/log/nginx/access.log \
  | sort -k2 -rn \
  | head -20

# cut: kolom sederhana
cut -d: -f1,3 /etc/passwd
cut -c1-10 file.txt
 
# sort & uniq
sort file.txt | uniq
sort file.txt | uniq -c | sort -rn    # frequency count
 
# tr: translate/delete characters
echo "hello world" | tr ' ' '\n' | sort | uniq -c
cat file.txt | tr '[:upper:]' '[:lower:]'
 
# wc: word count
wc -l /var/log/syslog               # line count
grep -c "" file.txt                 # equivalent
 
# tee: split output
command | tee output.log | grep "error"

# Parse JSON API response
curl -s https://api.example.com/users | jq '.[] | {name, email}'
curl -s data.json | jq '.results | length'
curl -s data.json | jq -r '.items[].name'   # raw output