Linux 08 — Server Security Hardening

SSH is the primary remote access protocol. It is also the most attacked service on the internet. Securing it is the single most important hardening step.

# Edit the SSH server configuration
$ sudo vim /etc/ssh/sshd_config

# Key settings to change:

# Disable root login (use sudo instead)
PermitRootLogin no

# Disable password authentication (use keys only — most important!)
PasswordAuthentication no
ChallengeResponseAuthentication no

# Disable empty passwords (always good practice)
PermitEmptyPasswords no

# Use a non-standard port (reduces automated scan noise)
Port 2222

# Allow only specific users (whitelist)
AllowUsers alice bob deployuser

# Restrict to specific groups
AllowGroups sshusers admins

# Limit authentication attempts
MaxAuthTries 3

# Disconnect idle sessions after 10 minutes
ClientAliveInterval 600
ClientAliveCountMax 0

# Disable unused authentication methods
GSSAPIAuthentication no
X11Forwarding no
AllowTcpForwarding no          # disable unless needed
AllowAgentForwarding no        # disable unless needed

# Log level for audit trail
LogLevel VERBOSE

# Restrict SSH protocol version (already v2 only in modern OpenSSH)
Protocol 2

# Limit who can use SSH (PAM)
UsePAM yes

# Apply changes — always validate before reloading!
$ sudo sshd -t                  # test configuration for syntax errors
$ sudo systemctl reload sshd    # reload (not restart — don't lock yourself out!)

# Keep your current session open while testing with a new session
# Only close current session after confirming the new one works

# Generate SSH key pair (on YOUR LOCAL machine, not the server)
$ ssh-keygen -t ed25519 -C "alice@company.com"    # modern, recommended
$ ssh-keygen -t rsa -b 4096 -C "alice@company.com"  # RSA 4096 (older systems)
# -t = type, -b = bits, -C = comment

# This creates two files:
# ~/.ssh/id_ed25519      — PRIVATE KEY (never share this!)
# ~/.ssh/id_ed25519.pub  — Public key (safe to share)

# Copy public key to server
$ ssh-copy-id -i ~/.ssh/id_ed25519.pub user@server     # easiest method
$ ssh-copy-id -i ~/.ssh/id_ed25519.pub -p 2222 user@server  # custom port

# Manual method (if ssh-copy-id not available)
$ cat ~/.ssh/id_ed25519.pub | ssh user@server "mkdir -p ~/.ssh && cat >> ~/.ssh/authorized_keys"

# On the server — ensure correct permissions (CRITICAL)
$ chmod 700 ~/.ssh
$ chmod 600 ~/.ssh/authorized_keys
$ chown -R $USER:$USER ~/.ssh

# View authorized keys
$ cat ~/.ssh/authorized_keys

# Restrict what a key can do in authorized_keys (add before key)
# no-pty,no-port-forwarding,command="/usr/bin/rsync --server" ssh-ed25519 AAAA...

# Test key-based login
$ ssh -i ~/.ssh/id_ed25519 user@server

# Add to ~/.ssh/config for convenience
Host myserver
    HostName 192.168.1.100
    Port 2222
    User alice
    IdentityFile ~/.ssh/id_ed25519
    IdentitiesOnly yes          # only use specified key

UFW is the user-friendly frontend to iptables on Ubuntu/Debian. It lets you set firewall rules with simple commands.

# Check UFW status
$ sudo ufw status
$ sudo ufw status verbose        # detailed output with rules
$ sudo ufw status numbered       # numbered rules (easier to delete)

# Set default policies (deny all incoming, allow all outgoing)
$ sudo ufw default deny incoming
$ sudo ufw default allow outgoing

# Allow services BEFORE enabling UFW
$ sudo ufw allow 22/tcp          # SSH (default port)
$ sudo ufw allow 2222/tcp        # SSH on custom port
$ sudo ufw allow 80/tcp          # HTTP
$ sudo ufw allow 443/tcp         # HTTPS
$ sudo ufw allow 80              # same — both tcp and udp
$ sudo ufw allow 'Nginx Full'    # allow nginx profile (80 + 443)
$ sudo ufw allow 3306/tcp        # MySQL (consider restricting by IP)

# Enable UFW
$ sudo ufw enable                # WILL PROMPT — say yes
$ sudo ufw disable               # disable firewall (leaves rules)
$ sudo ufw reload                # reload rules

# Deny rules
$ sudo ufw deny 25/tcp           # block outgoing SMTP (prevent spam relay)
$ sudo ufw deny from 203.0.113.0/24  # block IP range

# Restrict by source IP
$ sudo ufw allow from 192.168.1.0/24 to any port 3306   # MySQL from LAN only
$ sudo ufw allow from 10.0.0.50 to any port 22           # SSH from specific IP only
$ sudo ufw allow from 10.0.0.0/8                         # allow entire subnet

# Delete rules
$ sudo ufw status numbered       # see rule numbers
$ sudo ufw delete 3              # delete rule #3
$ sudo ufw delete allow 80       # delete by rule description

# Rate limiting (protect against brute force)
$ sudo ufw limit 22/tcp          # limit SSH connections (6 per 30 seconds)
$ sudo ufw limit 2222/tcp        # limit SSH on custom port

# View rules
$ sudo ufw show raw              # raw iptables rules
$ sudo ufw show added            # rules that have been added

Available UFW Application Profiles

$ sudo ufw app list              # list available profiles
$ sudo ufw app info 'Nginx Full' # show what profile allows
$ sudo ufw allow 'OpenSSH'       # allow from profile

# Install fail2ban
$ sudo apt install fail2ban      # Debian/Ubuntu
$ sudo yum install fail2ban      # CentOS/RHEL

# Start and enable
$ sudo systemctl enable --now fail2ban

# Configuration — NEVER edit /etc/fail2ban/jail.conf directly
# Create a local override file:
$ sudo cp /etc/fail2ban/jail.conf /etc/fail2ban/jail.local
# Or create specific jail override:
$ sudo vim /etc/fail2ban/jail.local

[DEFAULT]
# Ban host for 1 hour after 5 failures within 10 minutes
bantime  = 3600
findtime = 600
maxretry = 5

# Never ban these IPs (your own IP, monitoring servers)
ignoreip = 127.0.0.1/8 ::1 192.168.1.0/24 YOUR.OFFICE.IP.HERE

# Email notification on ban
destemail = admin@example.com
sendername = Fail2Ban Alert
mta = sendmail
action = %(action_mwl)s    # ban + email + whois + log lines

[sshd]
enabled  = true
port     = 2222            # your custom SSH port
logpath  = %(sshd_log)s
maxretry = 3               # stricter for SSH
bantime  = 86400           # ban SSH attackers for 24 hours

[nginx-http-auth]
enabled = true
port    = http,https
logpath = /var/log/nginx/error.log

[nginx-botsearch]
enabled  = true
port     = http,https
logpath  = /var/log/nginx/access.log
maxretry = 2

$ sudo systemctl restart fail2ban

# Monitor fail2ban
$ sudo fail2ban-client status                    # overall status
$ sudo fail2ban-client status sshd               # SSH jail status
$ sudo fail2ban-client set sshd unbanip 1.2.3.4  # unban an IP
$ sudo fail2ban-client banned                    # list all banned IPs

# View ban log
$ sudo journalctl -u fail2ban -f
$ sudo tail -f /var/log/fail2ban.log

# Install and start auditd
$ sudo apt install auditd audispd-plugins    # Debian/Ubuntu
$ sudo systemctl enable --now auditd

# List current audit rules
$ sudo auditctl -l

# Add audit rules
# Watch a file for read/write access
$ sudo auditctl -w /etc/passwd -p rwxa -k passwd_changes
$ sudo auditctl -w /etc/sudoers -p rwxa -k sudoers_changes
$ sudo auditctl -w /etc/ssh/sshd_config -p rwa -k sshd_config

# Watch a directory
$ sudo auditctl -w /var/www/html -p w -k webfile_changes

# Persistent rules (survive reboot)
$ sudo vim /etc/audit/rules.d/hardening.rules

# Delete all previous rules
-D

# Set buffer size
-b 8192

# Failure mode (1=printk, 2=panic)
-f 1

# Watch sensitive files
-w /etc/passwd -p wa -k identity
-w /etc/group -p wa -k identity
-w /etc/shadow -p wa -k identity
-w /etc/sudoers -p wa -k sudoers
-w /etc/ssh/sshd_config -p wa -k sshd_config

# Watch for su/sudo usage
-w /bin/su -p x -k priv_esc
-w /usr/bin/sudo -p x -k priv_esc

# Watch for cron changes
-w /etc/crontab -p wa -k cron
-w /etc/cron.d/ -p wa -k cron

# Log all failed system calls (performance impact — use selectively)
# -a always,exit -F arch=b64 -S all -F success=0 -k syscall_failures

$ sudo service auditd restart

# Search audit logs
$ sudo ausearch -k passwd_changes           # search by key
$ sudo ausearch -k identity -ts recent      # recent events
$ sudo ausearch -ua alice                   # audit events for user alice
$ sudo ausearch -m USER_LOGIN               # login events

# Generate reports
$ sudo aureport                             # summary report
$ sudo aureport --auth                      # authentication report
$ sudo aureport --login                     # login report
$ sudo aureport --failed                    # failed events
$ sudo aureport --file                      # file access report

# Install lynis
$ sudo apt install lynis         # Debian/Ubuntu
# Or from source for latest version:
$ git clone https://github.com/CISOfy/lynis
$ cd lynis

# Run a full system audit
$ sudo lynis audit system

# The audit checks hundreds of security settings and produces:
# - A hardening index score (0-100)
# - Suggestions for improvement
# - Warnings for critical issues

# Key sections in lynis output:
# [+] Boot and services
# [+] Kernel, memory, storage
# [+] Users, groups, authentication
# [+] Shells
# [+] File systems
# [+] Malware scanners
# [+] Firewalls
# [+] Cryptography
# [+] Logging and auditing
# [+] Software: web server (nginx/apache)
# [+] Databases
# [+] PHP
# [+] SSH support

# View report
$ sudo cat /var/log/lynis-report.dat

# Run without colors (for piping)
$ sudo lynis audit system --no-colors 2>&1 | tee lynis_report.txt

# Specific tests only
$ sudo lynis audit system --tests-from-group ssh
$ sudo lynis audit system --tests-from-group authentication

# Ubuntu/Debian — unattended-upgrades
$ sudo apt install unattended-upgrades
$ sudo dpkg-reconfigure -plow unattended-upgrades   # interactive setup

# Configuration
$ sudo vim /etc/apt/apt.conf.d/50unattended-upgrades

Unattended-Upgrade::Allowed-Origins {
    "${distro_id}:${distro_codename}-security";
    // "${distro_id}:${distro_codename}-updates";  // optional
};

// Automatically remove unused kernels
Unattended-Upgrade::Remove-Unused-Kernels "true";

// Automatically reboot if required
Unattended-Upgrade::Automatic-Reboot "false";        // change to true if desired
Unattended-Upgrade::Automatic-Reboot-Time "02:00";   // reboot at 2am if needed

// Email notification
Unattended-Upgrade::Mail "admin@example.com";
Unattended-Upgrade::MailReport "on-change";

# Enable auto-updates
$ sudo vim /etc/apt/apt.conf.d/20auto-upgrades

APT::Periodic::Update-Package-Lists "1";        // daily index update
APT::Periodic::Download-Upgradeable-Packages "1";
APT::Periodic::AutocleanInterval "7";
APT::Periodic::Unattended-Upgrade "1";           // daily auto-upgrade

# Test dry run
$ sudo unattended-upgrades --dry-run --debug

# CentOS/RHEL — automatic security updates
$ sudo dnf install dnf-automatic
$ sudo vim /etc/dnf/automatic.conf
# Set: apply_updates = yes
# Set: upgrade_type = security
$ sudo systemctl enable --now dnf-automatic.timer

AppArmor (Ubuntu default)

# AppArmor confines programs to a limited set of resources
$ sudo apt install apparmor apparmor-utils

# Check status
$ sudo apparmor_status
$ sudo aa-status

# AppArmor modes:
# enforce = policy violations are blocked and logged
# complain = violations are logged but NOT blocked (learning mode)

# Manage profiles
$ sudo aa-enforce /etc/apparmor.d/usr.sbin.nginx    # enforce nginx profile
$ sudo aa-complain /etc/apparmor.d/usr.sbin.mysqld  # complain mode for learning
$ sudo aa-disable /etc/apparmor.d/usr.sbin.tcpdump  # disable profile

# List loaded profiles
$ sudo aa-status | grep "profiles"

# Generate profile for new application (interactive)
$ sudo aa-genprof /usr/local/bin/myapp

SELinux (RHEL/CentOS default)

# Check SELinux status
$ sestatus
$ getenforce                    # Enforcing, Permissive, or Disabled

# Set SELinux mode
$ sudo setenforce 1             # Enforcing (temporarily)
$ sudo setenforce 0             # Permissive (temporarily)

# Permanent mode — /etc/selinux/config
SELINUX=enforcing               # enforcing | permissive | disabled

# View SELinux contexts
$ ls -Z /var/www/html/          # file contexts
$ ps auxZ | grep nginx          # process context
$ id -Z                         # your context

# Fix file contexts
$ sudo restorecon -Rv /var/www/html/    # restore default contexts
$ sudo chcon -t httpd_sys_content_t /var/www/html/file.html

# Allow nginx to connect to network (common SELinux issue)
$ sudo setsebool -P httpd_can_network_connect 1
$ sudo setsebool -P httpd_can_network_connect_db 1

# AIDE — Advanced Intrusion Detection Environment
# Creates a database of file hashes, detects unauthorized changes

$ sudo apt install aide         # Debian/Ubuntu

# Configure what to monitor
$ sudo vim /etc/aide/aide.conf
# Default config is usually fine for most systems

# Initialize database (takes a while — scans all specified files)
$ sudo aideinit
$ sudo cp /var/lib/aide/aide.db.new /var/lib/aide/aide.db

# Check for changes (run regularly via cron)
$ sudo aide --check

# Update database after legitimate changes
$ sudo aide --update
$ sudo mv /var/lib/aide/aide.db.new /var/lib/aide/aide.db

# Automate with cron
$ sudo crontab -e
# Add: 0 3 * * * /usr/bin/aide --check | mail -s "AIDE report $(hostname)" admin@example.com

# Critical system file permissions
# These should be verified and enforced regularly

# Authentication files
$ ls -la /etc/passwd             # should be: -rw-r--r-- (644)
$ ls -la /etc/shadow             # should be: -rw-r----- (640) root:shadow
$ ls -la /etc/group              # should be: -rw-r--r-- (644)
$ ls -la /etc/gshadow            # should be: -rw-r----- (640) root:shadow
$ ls -la /etc/sudoers            # should be: -r--r----- (440)

# Fix permissions if wrong
$ sudo chmod 644 /etc/passwd
$ sudo chmod 640 /etc/shadow
$ sudo chmod 644 /etc/group
$ sudo chmod 440 /etc/sudoers

# SSH directory permissions
$ ls -la ~/.ssh/
$ chmod 700 ~/.ssh
$ chmod 600 ~/.ssh/authorized_keys
$ chmod 600 ~/.ssh/id_rsa
$ chmod 644 ~/.ssh/id_rsa.pub
$ chmod 644 ~/.ssh/known_hosts

# Find files with dangerous permissions
# SUID files — can run as file owner (should be very few)
$ sudo find / -perm -4000 -type f 2>/dev/null | sort
# Common legitimate SUID: /usr/bin/passwd, /usr/bin/sudo, /usr/bin/su

# SGID files
$ sudo find / -perm -2000 -type f 2>/dev/null | sort

# World-writable files (potential backdoor locations)
$ sudo find / -perm -0002 -type f 2>/dev/null | grep -v proc | grep -v sys

# World-writable directories (should be limited)
$ sudo find / -perm -0002 -type d 2>/dev/null | grep -v proc

# Files with no owner (orphaned files)
$ sudo find / -nouser 2>/dev/null
$ sudo find / -nogroup 2>/dev/null

# Cron security
# Only root should write to system cron files
$ ls -la /etc/cron*
$ ls -la /var/spool/cron/

# Restrict cron to specific users
$ sudo vim /etc/cron.allow       # only listed users can use cron
$ sudo vim /etc/cron.deny        # listed users cannot use cron

# Check all crontabs for suspicious entries
$ sudo ls /var/spool/cron/crontabs/
$ sudo for user in $(cut -d: -f1 /etc/passwd); do crontab -l -u $user 2>/dev/null; done

# World-writable directory check (production servers)
$ sudo find / -xdev -type d -perm -0002 2>/dev/null | grep -v proc
# Legitimate: /tmp, /var/tmp, /run/lock — these should have sticky bit

# Check /tmp has sticky bit
$ ls -ld /tmp
drwxrwxrwt  # the 't' = sticky bit set (correct)

# Add sticky bit if missing
$ sudo chmod +t /tmp
$ sudo chmod +t /var/tmp

# Check for failed login attempts
$ sudo grep "Failed password" /var/log/auth.log | tail -20
$ sudo lastb | head -20                  # failed login attempts

# Check currently logged in users
$ who
$ w
$ last | head -20                        # successful logins

# Check for suspicious processes
$ ps aux | sort -rk 3 | head -20         # top CPU users
$ lsof -i -n -P | grep ESTABLISHED       # all network connections

# Check listening ports
$ ss -tlnp                               # TCP listening ports

# Check crontabs for all users
$ for u in $(cut -d: -f1 /etc/passwd); do
    sudo crontab -l -u "$u" 2>/dev/null | grep -v "^#" | grep . && echo "(user: $u)"
  done

# Check for rootkits (install rkhunter)
$ sudo apt install rkhunter chkrootkit
$ sudo rkhunter --update && sudo rkhunter --check
$ sudo chkrootkit

# Malware scanner (ClamAV)
$ sudo apt install clamav clamav-daemon
$ sudo freshclam                         # update definitions
$ sudo clamscan -r /home --infected      # scan home dirs, show infected only

# Check file changes since package install
$ sudo debsums -c 2>/dev/null | head -20  # Debian: verify package files

# Quick security check script
$ echo "=== Failed Logins ===" && sudo grep "Failed password" /var/log/auth.log | wc -l
$ echo "=== Listening Ports ===" && ss -tlnp | grep -v "127.0.0.1\|::1"
$ echo "=== SUID Files ===" && sudo find / -perm -4000 -type f 2>/dev/null | wc -l
$ echo "=== UFW Status ===" && sudo ufw status | head -5