#!/usr/bin/env bash
#
# nginx-log-report.sh: one report file from all nginx logs
#
# Reads every access log (the current one and the rotated ones, plain or .gz) and,
# if they exist and are readable, the nginx error logs, the SSH log and the firewall
# log, and writes ONE text file with statistics and lists of IP addresses:
#
#   * requests per day, status codes, requests per hour
#   * every external IP with first/last visit and status counts
#   * scanners: requests for typical attack paths, broken (non-HTTP) requests,
#     IPs with many 4xx answers, blocked (403) requests
#   * the /sharing app: uploads, downloads, logins, admin access
#   * user agents (bots, empty), top paths, top 404 paths
#   * nginx error log, failed SSH logins, port scan suspects (firewall log)
#
# It only reads, nothing is changed. The report contains IP addresses: it is
# created readable for the owner only (chmod 600).
#
# Usage:   sudo bash nginx-log-report.sh [options]
#   -d DIR     directory of the nginx logs               (default /var/log/nginx)
#   -o FILE    report file                               (default ./nginx-report-DATE-TIME.txt)
#   -n N       length of the "top" lists                 (default 30)
#   -t N       4xx answers from one IP to be listed      (default 20)
#   -S DATE    only entries since DATE (YYYY-MM-DD)
#   -i REGEX   which IPs are internal (extended regex)   (default: private ranges, VPN 100.64/10)
#   -s REGEX   paths which look like a scan              (default: .env, wp-login, phpmyadmin, ...)
#   -a GLOB    SSH log(s)                                (default '/var/log/auth.log*')
#   -f GLOB    firewall log(s), lines with SRC= and DPT= (default '/var/log/ufw.log*')
#   -h         this help
#
# Needs: bash 4, awk (mawk and gawk both work), gzip, sort, find. "column" is used when present.
# Log format: nginx "combined" (an extra field at the end, like X-Forwarded-For, is fine).

set -u
umask 077

LOG_DIR="/var/log/nginx"
OUT=""
TOP=30
THRESH=20
SINCE=""
AUTH_GLOB="/var/log/auth.log*"
FW_GLOB="/var/log/ufw.log*"
# environment variables, so that awk reads the regular expressions unchanged
export INTERNAL_RE='^(10\.|192\.168\.|127\.|172\.(1[6-9]|2[0-9]|3[01])\.|100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\.|::1$|[fF][cCdD][0-9a-fA-F]*:|[fF][eE]80:)'
export SCAN_RE='(\.env|\.git(/|$)|wp-(login|admin|content|includes)|xmlrpc|phpmyadmin|phpunit|eval-stdin|cgi-bin|actuator|boaform|HNAP1|\.php|\.asp|\.jsp|/etc/passwd|\.\./|manager/html|jndi|\.aws|\.DS_Store|autodiscover|/shell|\.sql$|\.bak$)'
export BOT_RE='(bot|crawl|spider|slurp|scrap|facebookexternalhit|fetcher|gpt|claude|perplexity|ahrefs|semrush|petal|yandex|baidu|python-requests|curl/|wget|go-http-client|java/|libwww|okhttp|httpclient|node-fetch|axios|masscan|zgrab|nmap|nikto|sqlmap|censys|shodan|expanse)'

usage() { sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//'; }

while getopts "d:o:n:t:S:i:s:a:f:h" opt; do
  case "$opt" in
    d) LOG_DIR="$OPTARG" ;;
    o) OUT="$OPTARG" ;;
    n) TOP="$OPTARG" ;;
    t) THRESH="$OPTARG" ;;
    S) SINCE="$OPTARG" ;;
    i) INTERNAL_RE="$OPTARG" ;;
    s) SCAN_RE="$OPTARG" ;;
    a) AUTH_GLOB="$OPTARG" ;;
    f) FW_GLOB="$OPTARG" ;;
    h) usage; exit 0 ;;
    *) usage >&2; exit 2 ;;
  esac
done

TAB=$'\t'
[ -z "$OUT" ] && OUT="./nginx-report-$(date +%Y%m%d-%H%M).txt"

for n in "$TOP" "$THRESH"; do
  case "$n" in ''|*[!0-9]*) echo "n and t need a number" >&2; exit 2 ;; esac
done
if [ -n "$SINCE" ] && ! [[ "$SINCE" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
  echo "-S needs a date like 2026-11-01" >&2; exit 2
fi

TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT
TSV="$TMP/access.tsv"

# files that can be read from a glob or a name pattern in LOG_DIR
list_files() { # $1 = glob
  local f
  for f in $1; do [ -f "$f" ] && [ -r "$f" ] && printf '%s\n' "$f"; done
}
read_logs() { gzip -dcf -- "$@" 2>/dev/null; }
fmt_cols() { if command -v column >/dev/null 2>&1; then column -t -s "$TAB"; else cat; fi; }
# table HEADER < rows (tab separated)
table() { { printf '%s\n' "$1"; cat; } | fmt_cols; }
section() { printf '\n\n==== %s ====\n\n' "$1"; }

mapfile -t ACCESS_FILES < <(find "$LOG_DIR" -maxdepth 1 -type f -name 'access.log*' 2>/dev/null | sort)
READABLE=()
for f in "${ACCESS_FILES[@]+"${ACCESS_FILES[@]}"}"; do
  if [ -r "$f" ]; then READABLE+=("$f"); else echo "cannot read $f (run with sudo?)" >&2; fi
done
if [ "${#READABLE[@]}" -eq 0 ]; then
  echo "no readable access.log* in $LOG_DIR (use -d, or run with sudo)" >&2
  exit 1
fi

# ---- 1. normalize: ip, date, time, method, path (no query), status, bytes, user agent
read_logs "${READABLE[@]}" | awk -v since="$SINCE" '
BEGIN { OFS = "\t"; months = "JanFebMarAprMayJunJulAugSepOctNovDec" }
{
  n = split($0, f, "\"")
  if (n < 6) { bad++; next }
  split(f[1], h, " ")
  ip = h[1]
  t = substr(h[4], 2)
  if (split(t, d, "[/:]") < 6) { bad++; next }
  m = (index(months, d[2]) + 2) / 3
  date = sprintf("%s-%02d-%s", d[3], m, d[1])
  time = d[4] ":" d[5] ":" d[6]
  if (since != "" && date < since) next
  split(f[2], r, " ")
  method = r[1]; path = r[2]
  if (path == "") path = "-"
  q = index(path, "?"); if (q) path = substr(path, 1, q - 1)
  split(f[3], s, " ")
  ua = f[6]
  gsub(/\t/, " ", method); gsub(/\t/, " ", path); gsub(/\t/, " ", ua)
  print ip, date, time, method, path, s[1], s[2], ua
}
END { print bad + 0 > "/dev/stderr" }' > "$TSV" 2> "$TMP/bad.txt"
BAD_LINES="$(cat "$TMP/bad.txt")"
TOTAL="$(wc -l < "$TSV" | tr -d ' ')"
if [ "$TOTAL" -eq 0 ]; then echo "no entries found (check the log format and -S)" >&2; exit 1; fi

# one row per IP: class ip requests first last 2xx 3xx 4xx 5xx top-path
awk -F'\t' '
BEGIN { re = ENVIRON["INTERNAL_RE"] }
{
  ip = $1; c[ip]++
  dt = $2 " " $3
  if (!(ip in first) || dt < first[ip]) first[ip] = dt
  if (dt > last[ip]) last[ip] = dt
  k = substr($6, 1, 1)
  if (k == "2") s2[ip]++; else if (k == "3") s3[ip]++; else if (k == "4") s4[ip]++; else if (k == "5") s5[ip]++
  pc[ip SUBSEP $5]++
}
END {
  for (key in pc) { split(key, a, SUBSEP); if (pc[key] > best[a[1]]) { best[a[1]] = pc[key]; top[a[1]] = a[2] } }
  for (ip in c) printf "%s\t%s\t%d\t%s\t%s\t%d\t%d\t%d\t%d\t%s\n", (ip ~ re ? "internal" : "external"), ip, c[ip], first[ip], last[ip], s2[ip], s3[ip], s4[ip], s5[ip], top[ip]
}' "$TSV" | sort -t"$TAB" -k3,3nr > "$TMP/ips.tsv"

# by_ip 'awk condition': per IP count and status codes of the rows that match
by_ip() {
  awk -F'\t' 'BEGIN { re = ENVIRON["INTERNAL_RE"] } '"$1"' {
    n[$1]++
    if (index(c[$1], $6) == 0) c[$1] = c[$1] $6 " "
  } END { for (ip in n) printf "%s\t%s\t%d\t%s\n", (ip ~ re ? "internal" : "external"), ip, n[ip], c[ip] }' "$TSV" \
    | sort -t"$TAB" -k3,3nr | head -n "$TOP"
}

report() {
  echo "NGINX LOG REPORT"
  echo "created:    $(date '+%Y-%m-%d %H:%M:%S')   on $(hostname 2>/dev/null)"
  [ -n "$SINCE" ] && echo "since:      $SINCE"

  section "1. Summary"
  awk -F'\t' '
  { d = $2 " " $3; if (min == "" || d < min) min = d; if (d > max) max = d }
  END { printf "period:     %s  to  %s\n", min, max }' "$TSV"
  echo "requests:   $TOTAL"
  echo "IPs:        $(wc -l < "$TMP/ips.tsv" | tr -d ' ') ($(awk -F'\t' '$1=="external"' "$TMP/ips.tsv" | wc -l | tr -d ' ') external)"
  echo "skipped:    $BAD_LINES lines which are not in the expected log format"
  echo "files read:"
  printf '            %s\n' "${READABLE[@]}"

  section "2. Requests per day (errors = status 400 and above)"
  awk -F'\t' '{ c[$2]++; if ($6 >= 400) e[$2]++ } END { for (k in c) printf "%s\t%d\t%d\n", k, c[k], e[k] }' "$TSV" | sort | table $'date\trequests\terrors'

  section "3. Status codes"
  awk -F'\t' '{ c[$6]++ } END { for (k in c) printf "%s\t%d\n", k, c[k] }' "$TSV" | sort -t"$TAB" -k2,2nr | table $'status\trequests'

  section "4. Requests per hour of the day"
  awk -F'\t' '{ c[substr($3, 1, 2)]++ } END {
    for (i = 0; i < 24; i++) { k = sprintf("%02d", i); if (c[k] > mx) mx = c[k] }
    for (i = 0; i < 24; i++) { k = sprintf("%02d", i); bar = ""; w = (mx ? int(40 * c[k] / mx) : 0); for (j = 0; j < w; j++) bar = bar "#"; printf "%s:00\t%d\t%s\n", k, c[k] + 0, bar }
  }' "$TSV" | table $'hour\trequests\t'

  section "5. EVERY external IP (most requests first, at most 5000)"
  awk -F'\t' '$1 == "external" { print $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6 "\t" $7 "\t" $8 "\t" $9 "\t" $10 }' "$TMP/ips.tsv" | head -n 5000 \
    | table $'ip\trequests\tfirst\tlast\t2xx\t3xx\t4xx\t5xx\tmost requested path'

  section "6. Internal IPs (top $TOP)"
  awk -F'\t' '$1 == "internal" { print $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6 "\t" $7 "\t" $8 "\t" $9 "\t" $10 }' "$TMP/ips.tsv" | head -n "$TOP" \
    | table $'ip\trequests\tfirst\tlast\t2xx\t3xx\t4xx\t5xx\tmost requested path'

  section "7. Scanners: requests for typical attack paths (not /sharing/, at most 500 IPs)"
  awk -F'\t' '
  BEGIN { scan = ENVIRON["SCAN_RE"]; re = ENVIRON["INTERNAL_RE"] }
  $5 ~ /^\/sharing\// { next }
  $5 ~ scan {
    ip = $1; n[ip]++
    if (index(codes[ip], $6) == 0) codes[ip] = codes[ip] $6 " "
    if (ns[ip] < 3 && index(sample[ip], $5) == 0) { sample[ip] = sample[ip] (sample[ip] == "" ? "" : "  ") $5; ns[ip]++ }
  }
  END { for (ip in n) printf "%s\t%s\t%d\t%s\t%s\n", (ip ~ re ? "internal" : "external"), ip, n[ip], codes[ip], sample[ip] }' "$TSV" \
    | sort -t"$TAB" -k3,3nr | head -n 500 | table $'class\tip\trequests\tstatus\texample paths'

  section "8. Broken requests (not HTTP, e.g. TLS or binary data sent to the HTTP port)"
  by_ip '$5 !~ /^\// || $4 !~ /^[A-Z]+$/' | table $'class\tip\trequests\tstatus'

  section "9. IPs with $THRESH or more error answers (400, 401, 403, 404, 405, 408, 444)"
  awk -F'\t' -v th="$THRESH" '
  BEGIN { re = ENVIRON["INTERNAL_RE"] }
  { t[$1]++; if ($6 == 400 || $6 == 401 || $6 == 403 || $6 == 404 || $6 == 405 || $6 == 408 || $6 == 444) e[$1]++ }
  END { for (ip in e) if (e[ip] >= th) printf "%s\t%s\t%d\t%d\t%d%%\n", (ip ~ re ? "internal" : "external"), ip, e[ip], t[ip], 100 * e[ip] / t[ip] }' "$TSV" \
    | sort -t"$TAB" -k3,3nr | head -n 500 | table $'class\tip\terrors\tall requests\tshare'

  section "10. Blocked requests (403) per IP (top $TOP)"
  by_ip '$6 == 403' | table $'class\tip\tblocked\tstatus'
  echo
  echo "most blocked paths:"
  awk -F'\t' '$6 == 403 { c[$5]++ } END { for (p in c) printf "%d\t%s\n", c[p], p }' "$TSV" | sort -t"$TAB" -k1,1nr | head -n "$TOP" | table $'blocked\tpath'

  section "11. /sharing"
  awk -F'\t' '
  BEGIN { re = ENVIRON["INTERNAL_RE"] }
  $5 ~ /^\/sharing/ { t++; if ($1 ~ re) ti++; else te++ }
  END { printf "requests:   %d (internal %d, external %d)\n", t, ti, te }' "$TSV"
  echo
  echo "-- external IPs which reached /sharing (status 403 = blocked by nginx, 401 = the app asked for a login):"
  by_ip '$5 ~ /^\/sharing/ && $1 !~ ENVIRON["INTERNAL_RE"]' | table $'class\tip\trequests\tstatus'
  echo
  echo "-- uploads (a file was created: POST /sharing/files/ answered 201):"
  by_ip '$4 == "POST" && $5 ~ /^\/sharing\/files\/?$/ && $6 == 201' | table $'class\tip\tfiles\tstatus'
  echo
  echo "-- downloads (GET /sharing/files/<file> answered 200 or 206, previews included, thumbnails not):"
  by_ip '$4 == "GET" && $5 ~ /^\/sharing\/files\// && $5 !~ /\.thumb$/ && ($6 == 200 || $6 == 206)' | table $'class\tip\tdownloads\tstatus'
  echo
  echo "-- logins (POST /sharing/login: 200 = right password, 403 = wrong password):"
  by_ip '$4 == "POST" && $5 == "/sharing/login"' | table $'class\tip\tattempts\tstatus'
  echo
  echo "-- admin pages (/sharing/admin...):"
  by_ip '$5 ~ /^\/sharing\/admin/' | table $'class\tip\trequests\tstatus'

  section "12. User agents (top $TOP)"
  awk -F'\t' '{ c[$8]++ } END { for (u in c) printf "%d\t%s\n", c[u], u }' "$TSV" | sort -t"$TAB" -k1,1nr | head -n "$TOP" | table $'requests\tuser agent'
  echo
  echo "-- automated clients and bots (by name) and empty user agents:"
  awk -F'\t' '
  BEGIN { bot = ENVIRON["BOT_RE"] }
  { u = tolower($8); if (u == "-" || u == "") u = "(empty)"
    if (u == "(empty)" || u ~ bot) { c[$8 == "" || $8 == "-" ? "(empty)" : $8]++ } }
  END { for (u in c) printf "%d\t%s\n", c[u], u }' "$TSV" | sort -t"$TAB" -k1,1nr | head -n "$TOP" | table $'requests\tuser agent'

  section "13. Most requested paths (top $TOP)"
  awk -F'\t' '{ c[$5]++ } END { for (p in c) printf "%d\t%s\n", c[p], p }' "$TSV" | sort -t"$TAB" -k1,1nr | head -n "$TOP" | table $'requests\tpath'
  echo
  echo "-- most requested paths which do not exist (404):"
  awk -F'\t' '$6 == 404 { c[$5]++ } END { for (p in c) printf "%d\t%s\n", c[p], p }' "$TSV" | sort -t"$TAB" -k1,1nr | head -n "$TOP" | table $'requests\tpath'

  section "14. nginx error log"
  mapfile -t ERR_FILES < <(find "$LOG_DIR" -maxdepth 1 -type f -name 'error.log*' -readable 2>/dev/null | sort)
  if [ "${#ERR_FILES[@]}" -eq 0 ]; then
    echo "(no readable error.log* in $LOG_DIR)"
  else
    read_logs "${ERR_FILES[@]}" > "$TMP/error.txt"
    echo "files: ${ERR_FILES[*]}"
    echo "lines: $(wc -l < "$TMP/error.txt" | tr -d ' ')"
    echo "TLS handshake failures: $(grep -c 'SSL_do_handshake() failed' "$TMP/error.txt")"
    echo
    echo "-- levels:"
    awk '{ print $3 }' "$TMP/error.txt" | sort | uniq -c | sort -rn | head -n 10
    echo
    echo "-- kinds of messages (top $TOP):"
    sed -E 's/^[0-9]{4}\/[0-9]{2}\/[0-9]{2} [0-9:]{8} \[[a-z]+\] [0-9]+#[0-9]+: (\*[0-9]+ )?//; s/,? client: .*$//' "$TMP/error.txt" \
      | cut -c1-90 | sort | uniq -c | sort -rn | head -n "$TOP"
    echo
    echo "-- client IPs in the error log (top $TOP):"
    grep -o 'client: [0-9a-fA-F.:]*' "$TMP/error.txt" | awk '{ print $2 }' | sort | uniq -c | sort -rn | head -n "$TOP"
  fi

  section "15. Failed SSH logins (auth log)"
  mapfile -t AUTH_FILES < <(list_files "$AUTH_GLOB")
  if [ "${#AUTH_FILES[@]}" -eq 0 ]; then
    echo "(no readable $AUTH_GLOB, use -a for another file)"
  else
    read_logs "${AUTH_FILES[@]}" | grep -E 'Failed password|Invalid user|authentication failure|Connection closed by (invalid|authenticating) user' \
      | grep -oE '(from|rhost=)[ =]?[0-9a-fA-F.:]+' | sed -E 's/^(from|rhost=)[ =]?//' | sort | uniq -c | sort -rn | head -n "$TOP" \
      | awk '{ printf "%s\t%s\n", $1, $2 }' | table $'failed\tip'
  fi

  section "16. Port scan suspects (firewall log: one IP, many different destination ports)"
  mapfile -t FW_FILES < <(list_files "$FW_GLOB")
  if [ "${#FW_FILES[@]}" -eq 0 ]; then
    echo "(no readable $FW_GLOB. nginx does not see port scans: switch the firewall log on, e.g. 'sudo ufw logging medium', and use -f)"
  else
    read_logs "${FW_FILES[@]}" | grep -E 'SRC=[0-9a-fA-F:.]+ .*DPT=' | awk '
    { src = ""; dpt = ""
      for (i = 1; i <= NF; i++) { if ($i ~ /^SRC=/) src = substr($i, 5); else if ($i ~ /^DPT=/) dpt = substr($i, 5) }
      if (src == "") next
      n[src]++
      if (!((src SUBSEP dpt) in seen)) { seen[src SUBSEP dpt] = 1; ports[src]++; if (np[src] < 8) { plist[src] = plist[src] dpt " "; np[src]++ } }
    }
    END { for (s in n) printf "%s\t%d\t%d\t%s\t%s\n", s, n[s], ports[s], (ports[s] >= 10 ? "SCAN SUSPECT" : "-"), plist[s] }' \
      | sort -t"$TAB" -k3,3nr -k2,2nr | head -n "$TOP" | table $'ip\tpackets\tdifferent ports\tverdict\tfirst ports'
  fi

  printf '\n\n---- end of report ----\n'
}

report > "$OUT"
chmod 600 "$OUT" 2>/dev/null
echo "report written: $OUT  ($TOTAL requests, $(wc -l < "$OUT" | tr -d ' ') lines)"