diff --git a/scripts/nginx-log-report.sh b/scripts/nginx-log-report.sh new file mode 100644 index 0000000..86dd7c3 --- /dev/null +++ b/scripts/nginx-log-report.sh @@ -0,0 +1,309 @@ +#!/usr/bin/env bash +# +# nginx-log-report.sh: one report file from all nginx logs +# +# Reads every access log (the current one and the rotated ones, plain or .gz) and, +# if they exist and are readable, the nginx error logs, the SSH log and the firewall +# log, and writes ONE text file with statistics and lists of IP addresses: +# +# * requests per day, status codes, requests per hour +# * every external IP with first/last visit and status counts +# * scanners: requests for typical attack paths, broken (non-HTTP) requests, +# IPs with many 4xx answers, blocked (403) requests +# * the /sharing app: uploads, downloads, logins, admin access +# * user agents (bots, empty), top paths, top 404 paths +# * nginx error log, failed SSH logins, port scan suspects (firewall log) +# +# It only reads, nothing is changed. The report contains IP addresses: it is +# created readable for the owner only (chmod 600). +# +# Usage: sudo bash nginx-log-report.sh [options] +# -d DIR directory of the nginx logs (default /var/log/nginx) +# -o FILE report file (default ./nginx-report-DATE-TIME.txt) +# -n N length of the "top" lists (default 30) +# -t N 4xx answers from one IP to be listed (default 20) +# -S DATE only entries since DATE (YYYY-MM-DD) +# -i REGEX which IPs are internal (extended regex) (default: private ranges, VPN 100.64/10) +# -s REGEX paths which look like a scan (default: .env, wp-login, phpmyadmin, ...) +# -a GLOB SSH log(s) (default '/var/log/auth.log*') +# -f GLOB firewall log(s), lines with SRC= and DPT= (default '/var/log/ufw.log*') +# -h this help +# +# Needs: bash 4, awk (mawk and gawk both work), gzip, sort, find. "column" is used when present. +# Log format: nginx "combined" (an extra field at the end, like X-Forwarded-For, is fine). + +set -u +umask 077 + +LOG_DIR="/var/log/nginx" +OUT="" +TOP=30 +THRESH=20 +SINCE="" +AUTH_GLOB="/var/log/auth.log*" +FW_GLOB="/var/log/ufw.log*" +# environment variables, so that awk reads the regular expressions unchanged +export INTERNAL_RE='^(10\.|192\.168\.|127\.|172\.(1[6-9]|2[0-9]|3[01])\.|100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\.|::1$|[fF][cCdD][0-9a-fA-F]*:|[fF][eE]80:)' +export SCAN_RE='(\.env|\.git(/|$)|wp-(login|admin|content|includes)|xmlrpc|phpmyadmin|phpunit|eval-stdin|cgi-bin|actuator|boaform|HNAP1|\.php|\.asp|\.jsp|/etc/passwd|\.\./|manager/html|jndi|\.aws|\.DS_Store|autodiscover|/shell|\.sql$|\.bak$)' +export BOT_RE='(bot|crawl|spider|slurp|scrap|facebookexternalhit|fetcher|gpt|claude|perplexity|ahrefs|semrush|petal|yandex|baidu|python-requests|curl/|wget|go-http-client|java/|libwww|okhttp|httpclient|node-fetch|axios|masscan|zgrab|nmap|nikto|sqlmap|censys|shodan|expanse)' + +usage() { sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//'; } + +while getopts "d:o:n:t:S:i:s:a:f:h" opt; do + case "$opt" in + d) LOG_DIR="$OPTARG" ;; + o) OUT="$OPTARG" ;; + n) TOP="$OPTARG" ;; + t) THRESH="$OPTARG" ;; + S) SINCE="$OPTARG" ;; + i) INTERNAL_RE="$OPTARG" ;; + s) SCAN_RE="$OPTARG" ;; + a) AUTH_GLOB="$OPTARG" ;; + f) FW_GLOB="$OPTARG" ;; + h) usage; exit 0 ;; + *) usage >&2; exit 2 ;; + esac +done + +TAB=$'\t' +[ -z "$OUT" ] && OUT="./nginx-report-$(date +%Y%m%d-%H%M).txt" + +for n in "$TOP" "$THRESH"; do + case "$n" in ''|*[!0-9]*) echo "n and t need a number" >&2; exit 2 ;; esac +done +if [ -n "$SINCE" ] && ! [[ "$SINCE" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then + echo "-S needs a date like 2026-11-01" >&2; exit 2 +fi + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +TSV="$TMP/access.tsv" + +# files that can be read from a glob or a name pattern in LOG_DIR +list_files() { # $1 = glob + local f + for f in $1; do [ -f "$f" ] && [ -r "$f" ] && printf '%s\n' "$f"; done +} +read_logs() { gzip -dcf -- "$@" 2>/dev/null; } +fmt_cols() { if command -v column >/dev/null 2>&1; then column -t -s "$TAB"; else cat; fi; } +# table HEADER < rows (tab separated) +table() { { printf '%s\n' "$1"; cat; } | fmt_cols; } +section() { printf '\n\n==== %s ====\n\n' "$1"; } + +mapfile -t ACCESS_FILES < <(find "$LOG_DIR" -maxdepth 1 -type f -name 'access.log*' 2>/dev/null | sort) +READABLE=() +for f in "${ACCESS_FILES[@]+"${ACCESS_FILES[@]}"}"; do + if [ -r "$f" ]; then READABLE+=("$f"); else echo "cannot read $f (run with sudo?)" >&2; fi +done +if [ "${#READABLE[@]}" -eq 0 ]; then + echo "no readable access.log* in $LOG_DIR (use -d, or run with sudo)" >&2 + exit 1 +fi + +# ---- 1. normalize: ip, date, time, method, path (no query), status, bytes, user agent +read_logs "${READABLE[@]}" | awk -v since="$SINCE" ' +BEGIN { OFS = "\t"; months = "JanFebMarAprMayJunJulAugSepOctNovDec" } +{ + n = split($0, f, "\"") + if (n < 6) { bad++; next } + split(f[1], h, " ") + ip = h[1] + t = substr(h[4], 2) + if (split(t, d, "[/:]") < 6) { bad++; next } + m = (index(months, d[2]) + 2) / 3 + date = sprintf("%s-%02d-%s", d[3], m, d[1]) + time = d[4] ":" d[5] ":" d[6] + if (since != "" && date < since) next + split(f[2], r, " ") + method = r[1]; path = r[2] + if (path == "") path = "-" + q = index(path, "?"); if (q) path = substr(path, 1, q - 1) + split(f[3], s, " ") + ua = f[6] + gsub(/\t/, " ", method); gsub(/\t/, " ", path); gsub(/\t/, " ", ua) + print ip, date, time, method, path, s[1], s[2], ua +} +END { print bad + 0 > "/dev/stderr" }' > "$TSV" 2> "$TMP/bad.txt" +BAD_LINES="$(cat "$TMP/bad.txt")" +TOTAL="$(wc -l < "$TSV" | tr -d ' ')" +if [ "$TOTAL" -eq 0 ]; then echo "no entries found (check the log format and -S)" >&2; exit 1; fi + +# one row per IP: class ip requests first last 2xx 3xx 4xx 5xx top-path +awk -F'\t' ' +BEGIN { re = ENVIRON["INTERNAL_RE"] } +{ + ip = $1; c[ip]++ + dt = $2 " " $3 + if (!(ip in first) || dt < first[ip]) first[ip] = dt + if (dt > last[ip]) last[ip] = dt + k = substr($6, 1, 1) + if (k == "2") s2[ip]++; else if (k == "3") s3[ip]++; else if (k == "4") s4[ip]++; else if (k == "5") s5[ip]++ + pc[ip SUBSEP $5]++ +} +END { + for (key in pc) { split(key, a, SUBSEP); if (pc[key] > best[a[1]]) { best[a[1]] = pc[key]; top[a[1]] = a[2] } } + for (ip in c) printf "%s\t%s\t%d\t%s\t%s\t%d\t%d\t%d\t%d\t%s\n", (ip ~ re ? "internal" : "external"), ip, c[ip], first[ip], last[ip], s2[ip], s3[ip], s4[ip], s5[ip], top[ip] +}' "$TSV" | sort -t"$TAB" -k3,3nr > "$TMP/ips.tsv" + +# by_ip 'awk condition': per IP count and status codes of the rows that match +by_ip() { + awk -F'\t' 'BEGIN { re = ENVIRON["INTERNAL_RE"] } '"$1"' { + n[$1]++ + if (index(c[$1], $6) == 0) c[$1] = c[$1] $6 " " + } END { for (ip in n) printf "%s\t%s\t%d\t%s\n", (ip ~ re ? "internal" : "external"), ip, n[ip], c[ip] }' "$TSV" \ + | sort -t"$TAB" -k3,3nr | head -n "$TOP" +} + +report() { + echo "NGINX LOG REPORT" + echo "created: $(date '+%Y-%m-%d %H:%M:%S') on $(hostname 2>/dev/null)" + [ -n "$SINCE" ] && echo "since: $SINCE" + + section "1. Summary" + awk -F'\t' ' + { d = $2 " " $3; if (min == "" || d < min) min = d; if (d > max) max = d } + END { printf "period: %s to %s\n", min, max }' "$TSV" + echo "requests: $TOTAL" + echo "IPs: $(wc -l < "$TMP/ips.tsv" | tr -d ' ') ($(awk -F'\t' '$1=="external"' "$TMP/ips.tsv" | wc -l | tr -d ' ') external)" + echo "skipped: $BAD_LINES lines which are not in the expected log format" + echo "files read:" + printf ' %s\n' "${READABLE[@]}" + + section "2. Requests per day (errors = status 400 and above)" + awk -F'\t' '{ c[$2]++; if ($6 >= 400) e[$2]++ } END { for (k in c) printf "%s\t%d\t%d\n", k, c[k], e[k] }' "$TSV" | sort | table $'date\trequests\terrors' + + section "3. Status codes" + awk -F'\t' '{ c[$6]++ } END { for (k in c) printf "%s\t%d\n", k, c[k] }' "$TSV" | sort -t"$TAB" -k2,2nr | table $'status\trequests' + + section "4. Requests per hour of the day" + awk -F'\t' '{ c[substr($3, 1, 2)]++ } END { + for (i = 0; i < 24; i++) { k = sprintf("%02d", i); if (c[k] > mx) mx = c[k] } + for (i = 0; i < 24; i++) { k = sprintf("%02d", i); bar = ""; w = (mx ? int(40 * c[k] / mx) : 0); for (j = 0; j < w; j++) bar = bar "#"; printf "%s:00\t%d\t%s\n", k, c[k] + 0, bar } + }' "$TSV" | table $'hour\trequests\t' + + section "5. EVERY external IP (most requests first, at most 5000)" + awk -F'\t' '$1 == "external" { print $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6 "\t" $7 "\t" $8 "\t" $9 "\t" $10 }' "$TMP/ips.tsv" | head -n 5000 \ + | table $'ip\trequests\tfirst\tlast\t2xx\t3xx\t4xx\t5xx\tmost requested path' + + section "6. Internal IPs (top $TOP)" + awk -F'\t' '$1 == "internal" { print $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6 "\t" $7 "\t" $8 "\t" $9 "\t" $10 }' "$TMP/ips.tsv" | head -n "$TOP" \ + | table $'ip\trequests\tfirst\tlast\t2xx\t3xx\t4xx\t5xx\tmost requested path' + + section "7. Scanners: requests for typical attack paths (not /sharing/, at most 500 IPs)" + awk -F'\t' ' + BEGIN { scan = ENVIRON["SCAN_RE"]; re = ENVIRON["INTERNAL_RE"] } + $5 ~ /^\/sharing\// { next } + $5 ~ scan { + ip = $1; n[ip]++ + if (index(codes[ip], $6) == 0) codes[ip] = codes[ip] $6 " " + if (ns[ip] < 3 && index(sample[ip], $5) == 0) { sample[ip] = sample[ip] (sample[ip] == "" ? "" : " ") $5; ns[ip]++ } + } + END { for (ip in n) printf "%s\t%s\t%d\t%s\t%s\n", (ip ~ re ? "internal" : "external"), ip, n[ip], codes[ip], sample[ip] }' "$TSV" \ + | sort -t"$TAB" -k3,3nr | head -n 500 | table $'class\tip\trequests\tstatus\texample paths' + + section "8. Broken requests (not HTTP, e.g. TLS or binary data sent to the HTTP port)" + by_ip '$5 !~ /^\// || $4 !~ /^[A-Z]+$/' | table $'class\tip\trequests\tstatus' + + section "9. IPs with $THRESH or more error answers (400, 401, 403, 404, 405, 408, 444)" + awk -F'\t' -v th="$THRESH" ' + BEGIN { re = ENVIRON["INTERNAL_RE"] } + { t[$1]++; if ($6 == 400 || $6 == 401 || $6 == 403 || $6 == 404 || $6 == 405 || $6 == 408 || $6 == 444) e[$1]++ } + END { for (ip in e) if (e[ip] >= th) printf "%s\t%s\t%d\t%d\t%d%%\n", (ip ~ re ? "internal" : "external"), ip, e[ip], t[ip], 100 * e[ip] / t[ip] }' "$TSV" \ + | sort -t"$TAB" -k3,3nr | head -n 500 | table $'class\tip\terrors\tall requests\tshare' + + section "10. Blocked requests (403) per IP (top $TOP)" + by_ip '$6 == 403' | table $'class\tip\tblocked\tstatus' + echo + echo "most blocked paths:" + awk -F'\t' '$6 == 403 { c[$5]++ } END { for (p in c) printf "%d\t%s\n", c[p], p }' "$TSV" | sort -t"$TAB" -k1,1nr | head -n "$TOP" | table $'blocked\tpath' + + section "11. /sharing" + awk -F'\t' ' + BEGIN { re = ENVIRON["INTERNAL_RE"] } + $5 ~ /^\/sharing/ { t++; if ($1 ~ re) ti++; else te++ } + END { printf "requests: %d (internal %d, external %d)\n", t, ti, te }' "$TSV" + echo + echo "-- external IPs which reached /sharing (status 403 = blocked by nginx, 401 = the app asked for a login):" + by_ip '$5 ~ /^\/sharing/ && $1 !~ ENVIRON["INTERNAL_RE"]' | table $'class\tip\trequests\tstatus' + echo + echo "-- uploads (a file was created: POST /sharing/files/ answered 201):" + by_ip '$4 == "POST" && $5 ~ /^\/sharing\/files\/?$/ && $6 == 201' | table $'class\tip\tfiles\tstatus' + echo + echo "-- downloads (GET /sharing/files/ answered 200 or 206, previews included, thumbnails not):" + by_ip '$4 == "GET" && $5 ~ /^\/sharing\/files\// && $5 !~ /\.thumb$/ && ($6 == 200 || $6 == 206)' | table $'class\tip\tdownloads\tstatus' + echo + echo "-- logins (POST /sharing/login: 200 = right password, 403 = wrong password):" + by_ip '$4 == "POST" && $5 == "/sharing/login"' | table $'class\tip\tattempts\tstatus' + echo + echo "-- admin pages (/sharing/admin...):" + by_ip '$5 ~ /^\/sharing\/admin/' | table $'class\tip\trequests\tstatus' + + section "12. User agents (top $TOP)" + awk -F'\t' '{ c[$8]++ } END { for (u in c) printf "%d\t%s\n", c[u], u }' "$TSV" | sort -t"$TAB" -k1,1nr | head -n "$TOP" | table $'requests\tuser agent' + echo + echo "-- automated clients and bots (by name) and empty user agents:" + awk -F'\t' ' + BEGIN { bot = ENVIRON["BOT_RE"] } + { u = tolower($8); if (u == "-" || u == "") u = "(empty)" + if (u == "(empty)" || u ~ bot) { c[$8 == "" || $8 == "-" ? "(empty)" : $8]++ } } + END { for (u in c) printf "%d\t%s\n", c[u], u }' "$TSV" | sort -t"$TAB" -k1,1nr | head -n "$TOP" | table $'requests\tuser agent' + + section "13. Most requested paths (top $TOP)" + awk -F'\t' '{ c[$5]++ } END { for (p in c) printf "%d\t%s\n", c[p], p }' "$TSV" | sort -t"$TAB" -k1,1nr | head -n "$TOP" | table $'requests\tpath' + echo + echo "-- most requested paths which do not exist (404):" + awk -F'\t' '$6 == 404 { c[$5]++ } END { for (p in c) printf "%d\t%s\n", c[p], p }' "$TSV" | sort -t"$TAB" -k1,1nr | head -n "$TOP" | table $'requests\tpath' + + section "14. nginx error log" + mapfile -t ERR_FILES < <(find "$LOG_DIR" -maxdepth 1 -type f -name 'error.log*' -readable 2>/dev/null | sort) + if [ "${#ERR_FILES[@]}" -eq 0 ]; then + echo "(no readable error.log* in $LOG_DIR)" + else + read_logs "${ERR_FILES[@]}" > "$TMP/error.txt" + echo "files: ${ERR_FILES[*]}" + echo "lines: $(wc -l < "$TMP/error.txt" | tr -d ' ')" + echo "TLS handshake failures: $(grep -c 'SSL_do_handshake() failed' "$TMP/error.txt")" + echo + echo "-- levels:" + awk '{ print $3 }' "$TMP/error.txt" | sort | uniq -c | sort -rn | head -n 10 + echo + echo "-- kinds of messages (top $TOP):" + sed -E 's/^[0-9]{4}\/[0-9]{2}\/[0-9]{2} [0-9:]{8} \[[a-z]+\] [0-9]+#[0-9]+: (\*[0-9]+ )?//; s/,? client: .*$//' "$TMP/error.txt" \ + | cut -c1-90 | sort | uniq -c | sort -rn | head -n "$TOP" + echo + echo "-- client IPs in the error log (top $TOP):" + grep -o 'client: [0-9a-fA-F.:]*' "$TMP/error.txt" | awk '{ print $2 }' | sort | uniq -c | sort -rn | head -n "$TOP" + fi + + section "15. Failed SSH logins (auth log)" + mapfile -t AUTH_FILES < <(list_files "$AUTH_GLOB") + if [ "${#AUTH_FILES[@]}" -eq 0 ]; then + echo "(no readable $AUTH_GLOB, use -a for another file)" + else + read_logs "${AUTH_FILES[@]}" | grep -E 'Failed password|Invalid user|authentication failure|Connection closed by (invalid|authenticating) user' \ + | grep -oE '(from|rhost=)[ =]?[0-9a-fA-F.:]+' | sed -E 's/^(from|rhost=)[ =]?//' | sort | uniq -c | sort -rn | head -n "$TOP" \ + | awk '{ printf "%s\t%s\n", $1, $2 }' | table $'failed\tip' + fi + + section "16. Port scan suspects (firewall log: one IP, many different destination ports)" + mapfile -t FW_FILES < <(list_files "$FW_GLOB") + if [ "${#FW_FILES[@]}" -eq 0 ]; then + echo "(no readable $FW_GLOB. nginx does not see port scans: switch the firewall log on, e.g. 'sudo ufw logging medium', and use -f)" + else + read_logs "${FW_FILES[@]}" | grep -E 'SRC=[0-9a-fA-F:.]+ .*DPT=' | awk ' + { src = ""; dpt = "" + for (i = 1; i <= NF; i++) { if ($i ~ /^SRC=/) src = substr($i, 5); else if ($i ~ /^DPT=/) dpt = substr($i, 5) } + if (src == "") next + n[src]++ + if (!((src SUBSEP dpt) in seen)) { seen[src SUBSEP dpt] = 1; ports[src]++; if (np[src] < 8) { plist[src] = plist[src] dpt " "; np[src]++ } } + } + END { for (s in n) printf "%s\t%d\t%d\t%s\t%s\n", s, n[s], ports[s], (ports[s] >= 10 ? "SCAN SUSPECT" : "-"), plist[s] }' \ + | sort -t"$TAB" -k3,3nr -k2,2nr | head -n "$TOP" | table $'ip\tpackets\tdifferent ports\tverdict\tfirst ports' + fi + + printf '\n\n---- end of report ----\n' +} + +report > "$OUT" +chmod 600 "$OUT" 2>/dev/null +echo "report written: $OUT ($TOTAL requests, $(wc -l < "$OUT" | tr -d ' ') lines)"