Linux treats almost everything as text. Knowing how to read, search, transform, and process text files is the difference between a productive admin and one who has to write GUI tools for everything.
# cat — concatenate and display files $ cat file.txt # display entire file $ cat -n file.txt # show with line numbers $ cat -A file.txt # show non-printing chars (tabs as ^I, newlines as $) $ cat file1.txt file2.txt # display both files in sequence $ cat file1.txt file2.txt > combined.txt # concatenate into new file $ cat >> file.txt # append typed input to file (Ctrl+D to end) $ cat /dev/null > file.txt # empty a file without deleting it # less — full-featured pager (preferred over more) $ less /var/log/syslog # open file in pager $ less +G /var/log/syslog # open at end of file $ less -N file.txt # show line numbers $ less -S file.txt # don't wrap long lines # less navigation keys: # Space / PgDn — next page # b / PgUp — previous page # G — go to end of file # g — go to beginning # /pattern — search forward # ?pattern — search backward # n — next match # N — previous match # q — quit # more — simpler pager (less is better) $ more file.txt
# head — show beginning of file $ head file.txt # first 10 lines (default) $ head -n 20 file.txt # first 20 lines $ head -20 file.txt # same (shorthand) $ head -c 1024 file.txt # first 1024 bytes $ head -n -5 file.txt # everything EXCEPT last 5 lines # tail — show end of file $ tail file.txt # last 10 lines (default) $ tail -n 20 file.txt # last 20 lines $ tail -50 file.txt # shorthand $ tail -c 512 file.txt # last 512 bytes $ tail -n +5 file.txt # from line 5 to end # tail -f — FOLLOW file in real time (essential for log monitoring) $ tail -f /var/log/nginx/access.log # watch log as requests come in $ tail -f /var/log/syslog # watch system log $ tail -F /var/log/app.log # follow even if file is rotated $ tail -f *.log # follow multiple log files # Combine head and tail to extract a range of lines $ sed -n '10,20p' file.txt # lines 10-20 (more reliable) $ awk 'NR>=10 && NR<=20' file.txt # same with awk
grep (Global Regular Expression Print) searches files for matching lines. It is the single most useful text tool on Linux.
# Basic usage
$ grep "error" /var/log/syslog # find lines containing "error"
$ grep "404" /var/log/nginx/access.log # find 404 errors in access log
$ grep "alice" /etc/passwd # find user in passwd
# grep flags
$ grep -i "error" file.log # -i: case insensitive
$ grep -r "TODO" /srv/app/ # -r: recursive (search directory)
$ grep -l "TODO" /srv/app/*.py # -l: print filenames only
$ grep -n "error" file.log # -n: show line numbers
$ grep -v "DEBUG" file.log # -v: invert (exclude matches)
$ grep -c "error" file.log # -c: count matching lines
$ grep -w "log" file.txt # -w: whole word only (not "logger")
$ grep -x "exact line" file.txt # -x: whole line must match
$ grep -A 3 "error" file.log # -A: 3 lines after match
$ grep -B 3 "error" file.log # -B: 3 lines before match
$ grep -C 3 "error" file.log # -C: 3 lines before AND after
$ grep -o "192\.168\.[0-9]*\.[0-9]*" file.log # -o: only print match
# Extended regex with -E (or use egrep)
$ grep -E "ERROR|WARN|CRITICAL" app.log # OR — match any
$ grep -E "^2024-06" access.log # lines starting with date
$ grep -E "[0-9]{3}\.[0-9]{3}" file.txt # IP-like patterns
$ grep -E "GET|POST|PUT|DELETE" access.log # HTTP methods
# Perl regex with -P (powerful — requires PCRE)
$ grep -P "\d{4}-\d{2}-\d{2}" file.log # date format YYYY-MM-DD
$ grep -P "(?<=user: )\w+" file.log # lookbehind assertion
# Multiple patterns
$ grep -e "error" -e "warning" file.log # match either pattern
$ grep -f patterns.txt file.log # patterns from file
# Practical log analysis examples
$ grep "error" /var/log/nginx/error.log | grep -v "robots.txt" # exclude noise
$ grep -i "failed" /var/log/auth.log | tail -20 # last 20 failures
$ grep "192.168.1.100" /var/log/nginx/access.log | wc -l # count hits from IP
$ grep -r "password" /etc/ 2>/dev/null | grep -v ".pyc" # find password configs
sed processes text line by line. It is most commonly used for substitution but can do much more.
# Substitution: s/find/replace/flags $ sed 's/old/new/' file.txt # replace first occurrence per line $ sed 's/old/new/g' file.txt # replace ALL occurrences (g=global) $ sed 's/old/new/2' file.txt # replace 2nd occurrence per line $ sed 's/old/new/gi' file.txt # global + case insensitive $ sed 's/error/ERROR/g' app.log # uppercase ERROR # In-place editing (-i) $ sed -i 's/foo/bar/g' config.txt # edit file directly (NO backup) $ sed -i.bak 's/foo/bar/g' config.txt # edit with .bak backup # Addresses — specify which lines to operate on $ sed '5s/old/new/' file.txt # only line 5 $ sed '2,8s/old/new/' file.txt # lines 2-8 $ sed '/pattern/s/old/new/' file.txt # only lines matching pattern $ sed '/^#/d' config.txt # delete comment lines $ sed '/^$/d' file.txt # delete empty lines $ sed '1d' file.txt # delete first line $ sed '$d' file.txt # delete last line $ sed '1,5d' file.txt # delete lines 1-5 # Print specific lines (-n suppresses default output) $ sed -n '5p' file.txt # print line 5 only $ sed -n '5,10p' file.txt # print lines 5-10 $ sed -n '/pattern/p' file.txt # print matching lines (like grep) # Insert and append $ sed '3i\New line before line 3' file.txt # insert before line 3 $ sed '3a\New line after line 3' file.txt # append after line 3 $ sed '1i\#!/bin/bash' script.sh # prepend shebang # Multiple commands $ sed -e 's/foo/bar/' -e 's/baz/qux/' file.txt # multiple substitutions $ sed '/^#/d; /^$/d' file.txt # delete comments and blanks # Practical examples $ sed 's/127\.0\.0\.1/0.0.0.0/g' app.conf # change bind address $ sed -n '/\[2024-06-15\]/,/\[2024-06-16\]/p' app.log # extract date range $ cat /etc/hosts | sed 's/#.*//' # strip comments
awk is a complete programming language for text processing. Think of it as a tool that processes a file row by row, splitting each row into fields.
# Basic syntax: awk 'pattern { action }' file
# Print specific fields (tab/space separated by default)
$ awk '{print $1}' file.txt # print first field of each line
$ awk '{print $2, $4}' file.txt # print 2nd and 4th fields
$ awk '{print $NF}' file.txt # print LAST field
$ awk '{print $(NF-1)}' file.txt # print second-to-last field
$ awk '{print NR, $0}' file.txt # print line number + whole line
# Custom field separator
$ awk -F: '{print $1, $6}' /etc/passwd # ':' separator — user + home
$ awk -F, '{print $2}' data.csv # CSV — print column 2
$ awk -F'\t' '{print $3}' data.tsv # tab separator
$ awk 'BEGIN{FS=":"} {print $1}' /etc/passwd # FS in BEGIN block
# Built-in variables
# NR = current row (record) number
# NF = number of fields in current row
# FS = field separator (default: whitespace)
# RS = record separator (default: newline)
# OFS = output field separator (default: space)
# ORS = output record separator (default: newline)
# Patterns — process only matching lines
$ awk '/error/' file.log # print lines with "error"
$ awk '!/^#/' config.txt # print non-comment lines
$ awk '$3 > 1000' data.txt # lines where field 3 > 1000
$ awk 'NR >= 5 && NR <= 10' file.txt # lines 5-10
$ awk 'NF > 3' file.txt # lines with more than 3 fields
# BEGIN and END blocks
$ awk 'BEGIN{print "Start"} {print} END{print "End"}' file.txt
$ awk 'BEGIN{sum=0} {sum+=$1} END{print "Total:", sum}' numbers.txt
# Practical awk examples
# Sum column 5 of a log file where status is 200
$ awk '$9 == 200 {sum += $10} END {print "Total bytes:", sum}' access.log
# Count occurrences
$ awk '{count[$1]++} END {for(ip in count) print count[ip], ip}' access.log | sort -rn
# Process /etc/passwd
$ awk -F: '$7 != "/usr/sbin/nologin" {print $1, $6, $7}' /etc/passwd
# Print lines between two patterns
$ awk '/START/,/END/' file.txt
# Add line numbers to output
$ awk '{printf "%4d %s\n", NR, $0}' file.txt
# Calculate average
$ awk '{sum+=$1; count++} END {print "Avg:", sum/count}' numbers.txt
# cut is simpler than awk for basic column extraction # Cut by delimiter (-d) and field (-f) $ cut -d: -f1 /etc/passwd # usernames (field 1) $ cut -d: -f1,6 /etc/passwd # username and home dir $ cut -d: -f1-4 /etc/passwd # fields 1 through 4 $ cut -d, -f2 data.csv # CSV second column $ cut -d$'\t' -f3 data.tsv # tab-delimited third field # Cut by character position (-c) $ cut -c1-10 file.txt # characters 1-10 $ cut -c5- file.txt # from character 5 to end $ cut -c-20 file.txt # first 20 characters # Practical examples $ cut -d: -f1 /etc/group | sort # list all groups $ ps aux | cut -c1-80 # truncate ps output width $ date | cut -d' ' -f1-3 # print day, month, date
# sort — sort lines of text
$ sort file.txt # alphabetical sort
$ sort -n numbers.txt # numeric sort
$ sort -r file.txt # reverse order
$ sort -rn numbers.txt # reverse numeric
$ sort -u file.txt # sort + remove duplicates
$ sort -k2 file.txt # sort by field 2
$ sort -k2,2 -k1,1 file.txt # sort by field 2, then field 1
$ sort -t: -k3 -n /etc/passwd # sort passwd by UID (field 3)
$ sort -h file.txt # human-readable sort (1K, 1M, 1G)
$ sort -R file.txt # random order (shuffle)
# uniq — filter duplicate lines (input must be SORTED)
$ uniq file.txt # remove consecutive duplicates
$ uniq -c file.txt # count occurrences (prefix count)
$ uniq -d file.txt # print only duplicate lines
$ uniq -u file.txt # print only unique lines (no dups)
$ uniq -i file.txt # case insensitive
# Common pattern: sort then uniq
$ sort access.log | uniq -c | sort -rn | head -20 # top 20 most common lines
$ cat access.log | awk '{print $1}' | sort | uniq -c | sort -rn # top IPs
# wc — word count $ wc file.txt # lines words bytes $ wc -l file.txt # count lines only $ wc -w file.txt # count words only $ wc -c file.txt # count bytes $ wc -m file.txt # count characters (Unicode-aware) $ wc -l *.log # count lines in all .log files $ ls /etc/ | wc -l # count files in /etc # diff — compare files $ diff file1.txt file2.txt # show differences $ diff -u file1.txt file2.txt # unified format (more readable) $ diff -i file1.txt file2.txt # ignore case differences $ diff -w file1.txt file2.txt # ignore whitespace $ diff -r dir1/ dir2/ # compare directories recursively $ diff -y file1.txt file2.txt # side-by-side comparison # Apply a patch $ diff -u original.txt modified.txt > changes.patch $ patch original.txt < changes.patch # tr — translate or delete characters $ echo "hello world" | tr 'a-z' 'A-Z' # lowercase to uppercase $ echo "HELLO" | tr 'A-Z' 'a-z' # uppercase to lowercase $ tr -d '\r' < windows.txt > unix.txt # remove Windows carriage returns $ tr -d '[:space:]' < file.txt # remove all whitespace $ tr -s ' ' < file.txt # squeeze multiple spaces into one $ echo "hello" | tr 'aeiou' '*' # replace vowels with * $ tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c 16 # generate random password # tee — write to file AND stdout simultaneously $ command | tee output.txt # save output and display it $ command | tee -a output.txt # append to file (don't overwrite) $ ls -la | tee listing.txt | grep ".sh" # tee in middle of pipeline
The pipe | sends the stdout of one command to the stdin of the next. This is the Unix philosophy in action: small tools that do one thing well, connected together.
# Basic pipes
$ ls -la | less # page through directory listing
$ cat /etc/passwd | grep bash # find users with bash shell
$ ps aux | grep nginx # find nginx processes
# Multi-step log analysis pipeline
$ cat /var/log/nginx/access.log \
| grep " 500 " \ # filter 500 errors
| awk '{print $1}' \ # extract IP address
| sort \ # sort IPs
| uniq -c \ # count each IP
| sort -rn \ # sort by count (highest first)
| head -10 # show top 10
# Find the most common user agents
$ awk -F'"' '{print $6}' /var/log/nginx/access.log \
| sort | uniq -c | sort -rn | head -10
# Find lines with errors, extract timestamp, show unique timestamps
$ grep -i error /var/log/app.log \
| awk '{print $1}' \
| sort -u
# Count failed SSH attempts per IP
$ grep "Failed password" /var/log/auth.log \
| awk '{print $(NF-3)}' \
| sort | uniq -c | sort -rn | head -20
# Disk usage report
$ du -sh /var/* 2>/dev/null | sort -h | tail -20
vim is available on nearly every Linux system. Even if you prefer nano or other editors, you need to know enough vim to edit files in emergencies.
# Open files $ vim file.txt # open file $ vim +50 file.txt # open at line 50 $ vim +/pattern file.txt # open at first match of pattern $ vim file1.txt file2.txt # open multiple files
| Mode | How to Enter | Purpose |
|---|---|---|
| Normal | ESC | Navigate, run commands — default mode |
| Insert | i, a, o, O, I, A | Type and edit text |
| Visual | v, V, Ctrl+V | Select text for operations |
| Command | : (colon) | Save, quit, search, replace |
# Essential Normal mode commands i # insert before cursor a # append after cursor I # insert at beginning of line A # append at end of line o # open new line below O # open new line above ESC # return to normal mode h j k l # move left/down/up/right (or use arrow keys) w # next word b # previous word 0 # beginning of line $ # end of line gg # go to first line G # go to last line 50G # go to line 50 Ctrl+f # page forward Ctrl+b # page backward dd # delete (cut) current line 5dd # delete 5 lines yy # yank (copy) current line 5yy # copy 5 lines p # paste after cursor P # paste before cursor u # undo Ctrl+r # redo /pattern # search forward ?pattern # search backward n # next match N # previous match # Command mode (:) :w # write (save) file :q # quit (fails if unsaved changes) :wq # save and quit :q! # quit without saving (FORCE) :wq! # save and quit (force, for read-only files as root) :x # save and quit (only writes if changed) :50 # go to line 50 # Search and replace in command mode :%s/old/new/g # replace all occurrences in file :%s/old/new/gc # replace with confirmation :5,20s/old/new/g # replace in lines 5-20 :s/old/new/ # replace first on current line # Working with multiple files :e file2.txt # open another file :n # next file :prev # previous file :ls # list open buffers :split file.txt # horizontal split :vsplit file.txt # vertical split Ctrl+w w # switch between splits
:q! and press Enter. This force-quits without saving.
Real-world example: Analyze an nginx access log to find performance issues.
#!/bin/bash
# Nginx log analysis pipeline
LOGFILE="/var/log/nginx/access.log"
echo "=== Top 10 IPs by Request Count ==="
awk '{print $1}' "$LOGFILE" | sort | uniq -c | sort -rn | head -10
echo ""
echo "=== HTTP Status Code Distribution ==="
awk '{print $9}' "$LOGFILE" | sort | uniq -c | sort -rn
echo ""
echo "=== Slowest Requests (response time) ==="
# Assuming $NF contains response time in microseconds
awk '{print $NF, $7}' "$LOGFILE" | sort -rn | head -10
echo ""
echo "=== 404 Error Pages ==="
awk '$9 == 404 {print $7}' "$LOGFILE" | sort | uniq -c | sort -rn | head -20
echo ""
echo "=== Total Traffic (bytes) ==="
awk '{sum += $10} END {printf "Total: %.2f MB\n", sum/1024/1024}' "$LOGFILE"
echo ""
echo "=== Requests by Hour ==="
awk '{print $4}' "$LOGFILE" | cut -d: -f2 | sort | uniq -c