Linux Server Performance Tuning

Problem

Required Tools

Solution Steps

# Real-time CPU usage (1-second interval, 10 samples)
vmstat 1 10

# Output interpretation:
#  r: processes waiting for CPU (overloaded if > CPU core count)
# us: user space CPU (application)
# sy: system space CPU (kernel)
# wa: I/O wait CPU (suspect disk bottleneck)
# id: idle CPU

# Per-CPU detailed usage
mpstat -P ALL 1 5

# Top CPU-consuming processes
ps aux --sort=-%cpu | head -20

# Check context switching count
vmstat 1 5 | awk '{print $12, $13}'  # cs column

# Check and change CPU scheduler
cat /sys/block/sda/queue/scheduler
# [mq-deadline] kyber bfq none

# Increase priority of important processes (nice)
# -20 (highest) to 19 (lowest)
sudo renice -n -10 -p $(pgrep -f "node.*app.js")

# Check CPU frequency governor (set performance mode)
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
echo "performance" | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor

# Detailed memory usage
free -h
# Note: "available" column is the actual usable memory (including buff/cache)

# Per-process memory usage (RSS: actual physical memory)
ps aux --sort=-%mem | head -20

# Swap usage
swapon --show

# --- Memory kernel parameter tuning ---

# Adjust swappiness (default 60, server recommended: 10-20)
# Lower values prefer physical memory over swap
sudo sysctl vm.swappiness=10

# Cache pressure adjustment (default 100)
# Lower values keep inode/dentry cache longer
sudo sysctl vm.vfs_cache_pressure=50

# Dirty Pages settings (disk write frequency)
sudo sysctl vm.dirty_ratio=15
sudo sysctl vm.dirty_background_ratio=5

# Protect critical processes from OOM Killer
# -1000: never OOM Kill
echo -1000 | sudo tee /proc/$(pgrep -f "node.*app.js")/oom_score_adj

# Disable Transparent Huge Pages (recommended for DB servers)
echo never | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
echo never | sudo tee /sys/kernel/mm/transparent_hugepage/defrag

# Persistent settings: /etc/sysctl.conf
cat << 'EOF' | sudo tee -a /etc/sysctl.conf
vm.swappiness = 10
vm.vfs_cache_pressure = 50
vm.dirty_ratio = 15
vm.dirty_background_ratio = 5
EOF
sudo sysctl -p

# Disk I/O statistics (1-second interval)
iostat -xz 1 5

# Key metrics:
# await: average I/O wait time (ms) - bottleneck if > 10ms
# %util: disk utilization - saturated if > 80%
# r/s, w/s: read/write requests per second

# Find which process uses the most I/O
sudo iotop -o

# Change I/O scheduler
# SSD: none or mq-deadline (prevent unnecessary reordering)
# HDD: bfq or mq-deadline
echo "mq-deadline" | sudo tee /sys/block/sda/queue/scheduler

# Adjust readahead size (default 128KB)
# Increase for servers with heavy sequential reads
sudo blockdev --setra 2048 /dev/sda  # Increase to 1MB

# Filesystem mount option optimization (/etc/fstab)
# noatime: disable file access time recording (reduces I/O)
# nodiratime: disable directory access time recording

# Check current mount options
mount | grep "on / "

# /etc/fstab example
# /dev/sda1 / ext4 defaults,noatime,nodiratime 0 1

# Check and enable TRIM support (SSD)
sudo fstrim -v /
sudo systemctl enable fstrim.timer

# Check current network connection status
ss -s
# Connection count by TCP state
ss -tan | awk '{print $1}' | sort | uniq -c | sort -rn

# --- Increase file descriptor limits ---
ulimit -n       # soft limit
ulimit -Hn      # hard limit

# /etc/security/limits.conf
cat << 'EOF' | sudo tee -a /etc/security/limits.conf
* soft nofile 65535
* hard nofile 65535
root soft nofile 65535
root hard nofile 65535
EOF

# --- Network kernel parameter tuning ---
cat << 'EOF' | sudo tee -a /etc/sysctl.conf

# Increase TCP backlog (handle concurrent connections)
net.core.somaxconn = 65535
net.ipv4.tcp_max_syn_backlog = 65535
net.core.netdev_max_backlog = 65535

# Socket buffer sizes (improve network throughput)
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216

# TIME_WAIT socket management
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_tw_buckets = 2000000
net.ipv4.tcp_fin_timeout = 15

# TCP Keepalive (detect idle connections)
net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp_keepalive_probes = 5

# Expand available local port range
net.ipv4.ip_local_port_range = 1024 65535

# SYN Flood defense
net.ipv4.tcp_syncookies = 1
EOF

sudo sysctl -p

# List running services (enabled state)
systemctl list-unit-files --type=service --state=enabled

# Disable unnecessary services
sudo systemctl disable --now snapd.service
sudo systemctl disable --now bluetooth.service
sudo systemctl disable --now cups.service

# Set resource limits on systemd services
sudo systemctl edit myapp.service
# [Service]
# CPUQuota=80%
# MemoryMax=4G
# MemorySwapMax=0
# LimitNOFILE=65535
# LimitNPROC=4096
# Restart=always
# RestartSec=5

# Apply changes
sudo systemctl daemon-reload
sudo systemctl restart myapp.service

# Monitor resources via cgroup
systemd-cgtop

# Analyze boot time (identify slow services)
systemd-analyze blame | head -20

# Check kernel logs for errors
sudo dmesg -T --level=err,warn | tail -30

#!/bin/bash
# linux-perf-check.sh - Linux server performance check script

echo "============================================"
echo " Linux Server Performance Report"
echo " $(date)"
echo "============================================"

# 1. CPU status
echo -e "\n--- CPU ---"
echo "Load Average: $(uptime | awk -F'load average:' '{print $2}')"
echo "CPU Cores: $(nproc)"
mpstat 1 1 | tail -1 | awk '{printf "User: %s%% | System: %s%% | IOWait: %s%% | Idle: %s%%\n", $4, $6, $7, $13}'

# 2. Memory status
echo -e "\n--- Memory ---"
free -h | awk '/^Mem:/{printf "Total: %s | Used: %s | Available: %s\n", $2, $3, $7}'
echo "Swap Usage: $(free -h | awk '/^Swap:/{print $3}')"
echo "Swappiness: $(sysctl -n vm.swappiness)"

# 3. Disk I/O
echo -e "\n--- Disk I/O ---"
iostat -xz 1 1 | awk 'NR>6 && $1!="" {printf "%s - await: %sms, util: %s%%\n", $1, $10, $NF}'

# 4. Disk usage
echo -e "\n--- Disk Usage ---"
df -h | awk 'NR>1 && $5+0 > 70 {printf "WARNING: %s is %s full (%s)\n", $6, $5, $1}'

# 5. Network status
echo -e "\n--- Network ---"
echo "TCP Connections:"
ss -tan | awk 'NR>1 {print $1}' | sort | uniq -c | sort -rn | head -5
echo "Open Files: $(cat /proc/sys/fs/file-nr | awk '{print $1}') / $(cat /proc/sys/fs/file-max)"

# 6. Top processes
echo -e "\n--- Top 5 CPU Processes ---"
ps aux --sort=-%cpu | awk 'NR<=6 {printf "%-10s %5s%% CPU %5s%% MEM  %s\n", $1, $3, $4, $11}'

echo -e "\n--- Top 5 Memory Processes ---"
ps aux --sort=-%mem | awk 'NR<=6 {printf "%-10s %5s%% MEM %10s KB  %s\n", $1, $4, $6, $11}'

echo -e "\n============================================"
echo " Check Complete"
echo "============================================"

Core Code

Core sysctl settings for production Linux servers. Includes swap minimization, high concurrency support, TIME_WAIT management, and SYN flood defense.

# /etc/sysctl.conf - Core Linux server performance settings
# ================================================

# --- Memory ---
vm.swappiness = 10
vm.vfs_cache_pressure = 50
vm.dirty_ratio = 15
vm.dirty_background_ratio = 5

# --- Network (high concurrency) ---
net.core.somaxconn = 65535
net.ipv4.tcp_max_syn_backlog = 65535
net.core.netdev_max_backlog = 65535
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216

# --- TIME_WAIT / Keepalive ---
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_fin_timeout = 15
net.ipv4.tcp_keepalive_time = 300
net.ipv4.ip_local_port_range = 1024 65535

# --- Security ---
net.ipv4.tcp_syncookies = 1

# Apply: sudo sysctl -p

Common Mistakes

Related liminfo Services