Automating IP Traffic Analysis with Shell Tools and the IP2Location.io Bulk API

#!/usr/bin/env bash
set -Eeuo pipefail

# Store your IP2Location.io API key here.
API_KEY="YOUR_API_KEY"

# Directory served by Apache or Nginx.
REPORT_DIR="/var/www/html/ip-traffic-report"

# The API returns only the fields used by this report.
API_FIELDS="country_code,country_name,region_name,city_name,latitude,longitude,asn,as,isp,usage_type,is_proxy,proxy.is_vpn,proxy.is_tor,proxy.is_data_center,proxy.is_web_crawler,proxy.is_ai_crawler"

usage() {
    cat <<'EOF'
Usage:
  analyze-ip-traffic.sh LOG_FILE [LOG_FILE ...]

Examples:
  analyze-ip-traffic.sh /var/log/nginx/access.log
  analyze-ip-traffic.sh /var/log/apache2/access.log
  analyze-ip-traffic.sh /var/log/nginx/access.log.1.gz
EOF
}

fail() {
    printf 'Error: %s\n' "$*" >&2
    exit 1
}

for command_name in curl jq python3 split gzip; do
    command -v "$command_name" >/dev/null 2>&1 || \
        fail "$command_name is required."
done

[[ -n "$API_KEY" && "$API_KEY" != "YOUR_API_KEY" ]] || \
    fail "Replace YOUR_API_KEY near the top of the script."

(( $# > 0 )) || {
    usage
    exit 1
}

for log_file in "$@"; do
    [[ -r "$log_file" ]] || fail "Cannot read log file: $log_file"
done

umask 022
mkdir -p "$REPORT_DIR"

WORK_DIR="$(mktemp -d)"
trap 'rm -rf "$WORK_DIR"' EXIT

SNAPSHOT="$WORK_DIR/access.log"
ALL_PUBLIC_IPS="$WORK_DIR/all-public-ips.txt"
COUNTS_TSV="$WORK_DIR/ip-counts.tsv"
UNIQUE_IPS="$WORK_DIR/unique-ips.txt"
COUNTS_JSON="$WORK_DIR/ip-counts.json"
GEO_JSON="$WORK_DIR/geolocation.json"
RECORDS_JSON="$WORK_DIR/records.json"
CURL_CONFIG="$WORK_DIR/curl.conf"
REPORT_TMP="$REPORT_DIR/.index.html.tmp.$$"

# Keep the key out of the curl command line and process list.
printf 'header = "Authorization: Bearer %s"\n' "$API_KEY" > "$CURL_CONFIG"
chmod 600 "$CURL_CONFIG"

printf 'Reading %d log file(s)...\n' "$#"

for log_file in "$@"; do
    case "$log_file" in
        *.gz)
            gzip -cd -- "$log_file"
            ;;
        *)
            cat -- "$log_file"
            ;;
    esac

    # Prevent two files from joining on one line when a file has no final newline.
    printf '\n'
done > "$SNAPSHOT"

TOTAL_LOG_LINES="$(awk 'NF { count++ } END { print count + 0 }' "$SNAPSHOT")"

# Apache and Nginx combined logs normally place the client address first.
# Python validates both IPv4 and IPv6 and keeps only globally routable IPs.
awk 'NF { print $1 }' "$SNAPSHOT" |
python3 -c '
import ipaddress
import sys

for line in sys.stdin:
    candidate = line.strip().strip("[]").split(",", 1)[0].strip()

    try:
        address = ipaddress.ip_address(candidate)
    except ValueError:
        continue

    if address.is_global:
        print(address.compressed)
' > "$ALL_PUBLIC_IPS"

PUBLIC_REQUESTS="$(wc -l < "$ALL_PUBLIC_IPS" | tr -d ' ')"

(( PUBLIC_REQUESTS > 0 )) || \
    fail "No globally routable IPv4 or IPv6 addresses were found."

LC_ALL=C sort "$ALL_PUBLIC_IPS" |
uniq -c |
awk '{ print $2 "\t" $1 }' > "$COUNTS_TSV"

cut -f1 "$COUNTS_TSV" > "$UNIQUE_IPS"
UNIQUE_IP_COUNT="$(wc -l < "$UNIQUE_IPS" | tr -d ' ')"

jq -Rn '
    [
        inputs
        | select(length > 0)
        | split("\t")
        | { (.[0]): (.[1] | tonumber) }
    ]
    | add
' < "$COUNTS_TSV" > "$COUNTS_JSON"

printf 'Looking up %s unique public IP address(es)...\n' "$UNIQUE_IP_COUNT"

# The Bulk API accepts no more than 1,000 addresses in one request.
split -l 1000 -d -a 4 "$UNIQUE_IPS" "$WORK_DIR/batch_"

response_number=0

for batch_file in "$WORK_DIR"/batch_*; do
    [[ -e "$batch_file" ]] || continue

    response_number=$((response_number + 1))
    response_file="$WORK_DIR/response_${response_number}.json"
    batch_size="$(wc -l < "$batch_file" | tr -d ' ')"

    printf '  Sending batch %d (%s IPs)...\n' \
        "$response_number" \
        "$batch_size"

    if ! curl \
        --config "$CURL_CONFIG" \
        --silent \
        --show-error \
        --fail-with-body \
        --retry 3 \
        --retry-delay 2 \
        --retry-all-errors \
        --connect-timeout 10 \
        --max-time 120 \
        --request POST \
        --data-binary @"$batch_file" \
        "https://bulk.ip2location.io/?format=json&fields=${API_FIELDS}" \
        --output "$response_file"; then

        if [[ -s "$response_file" ]]; then
            cat "$response_file" >&2
            printf '\n' >&2
        fi

        fail "The Bulk API request failed for batch $response_number."
    fi

    jq -e 'type == "object"' "$response_file" >/dev/null || \
        fail "The API returned invalid JSON for batch $response_number."

    if jq -e 'has("error")' "$response_file" >/dev/null; then
        jq '.error' "$response_file" >&2
        fail "The API returned an error for batch $response_number."
    fi
done

# Merge all API response objects.
jq -s '
    reduce .[] as $response
    ({}; . * $response)
' "$WORK_DIR"/response_*.json > "$GEO_JSON"

# Convert the response object to an array and restore request counts.
jq --slurpfile counts "$COUNTS_JSON" '
    to_entries
    | map(
        . as $entry
        | $entry.value
        + {
            ip: $entry.key,
            request_count: ($counts[0][$entry.key] // 0)
          }
      )
' "$GEO_JSON" > "$RECORDS_JSON"

cp "$RECORDS_JSON" "$REPORT_DIR/traffic-data.json"

# Generate a detailed CSV file.
jq -r '
    (
        [
            "IP address",
            "Requests",
            "Country code",
            "Country",
            "Region",
            "City",
            "Latitude",
            "Longitude",
            "ASN",
            "AS",
            "ISP",
            "Usage type",
            "Is proxy",
            "VPN",
            "Tor",
            "Data center",
            "Web crawler",
            "AI crawler"
        ]
        | @csv
    ),
    (
        .[]
        | [
            .ip,
            .request_count,
            (.country_code // ""),
            (.country_name // ""),
            (.region_name // ""),
            (.city_name // ""),
            (.latitude // ""),
            (.longitude // ""),
            (.asn // ""),
            (.as // ""),
            (.isp // ""),
            (.usage_type // ""),
            (.is_proxy // false),
            (.proxy.is_vpn // false),
            (.proxy.is_tor // false),
            (.proxy.is_data_center // false),
            (.proxy.is_web_crawler // false),
            (.proxy.is_ai_crawler // false)
          ]
        | @csv
    )
' "$RECORDS_JSON" > "$REPORT_DIR/traffic-details.csv"

COUNTRY_COUNT="$(
    jq '[.[].country_code // empty] | unique | length' "$RECORDS_JSON"
)"

MAPPED_IP_COUNT="$(
    jq '[.[] | select((.latitude | type) == "number" and (.longitude | type) == "number")] | length' \
        "$RECORDS_JSON"
)"

PROXY_REQUESTS="$(
    jq '[.[] | select(.is_proxy == true) | .request_count] | add // 0' \
        "$RECORDS_JSON"
)"

GENERATED_AT="$(date --iso-8601=seconds)"

TOP_COUNTRIES_ROWS="$(
    jq -r '
        group_by(.country_code // "")
        | map({
            country: (.[0].country_name // "Unknown"),
            code: (.[0].country_code // "--"),
            requests: (map(.request_count) | add),
            unique_ips: length
          })
        | sort_by(-.requests)
        | .[:15][]
        | "<tr>"
          + "<td>\((.country) | @html)</td>"
          + "<td>\((.code) | @html)</td>"
          + "<td>\(.requests)</td>"
          + "<td>\(.unique_ips)</td>"
          + "</tr>"
    ' "$RECORDS_JSON"
)"

TOP_CITIES_ROWS="$(
    jq -r '
        group_by([
            .country_code // "",
            .region_name // "",
            .city_name // ""
        ])
        | map({
            location: (
                [
                    .[0].city_name,
                    .[0].region_name,
                    .[0].country_name
                ]
                | map(select(. != null and . != ""))
                | join(", ")
            ),
            requests: (map(.request_count) | add),
            unique_ips: length
          })
        | sort_by(-.requests)
        | .[:15][]
        | "<tr>"
          + "<td>\(((if .location == "" then "Unknown" else .location end)) | @html)</td>"
          + "<td>\(.requests)</td>"
          + "<td>\(.unique_ips)</td>"
          + "</tr>"
    ' "$RECORDS_JSON"
)"

TOP_IP_ROWS="$(
    jq -r '
        sort_by(-.request_count)
        | .[:20][]
        | "<tr>"
          + "<td><code>\((.ip) | @html)</code></td>"
          + "<td>\(.request_count)</td>"
          + "<td>\((.country_name // "Unknown") | @html)</td>"
          + "<td>\((.city_name // "") | @html)</td>"
          + "<td>\((.isp // "") | @html)</td>"
          + "</tr>"
    ' "$RECORDS_JSON"
)"

cat > "$REPORT_TMP" <<HTML
<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>IP Traffic Analysis Report</title>

  <link
    rel="stylesheet"
    href="https://unpkg.com/[email protected]/dist/leaflet.css">

  <link
    rel="stylesheet"
    href="https://unpkg.com/[email protected]/dist/MarkerCluster.css">

  <link
    rel="stylesheet"
    href="https://unpkg.com/[email protected]/dist/MarkerCluster.Default.css">

  <style>
    :root {
      color-scheme: light;
      --background: #f4f7fb;
      --card: #ffffff;
      --text: #172033;
      --muted: #64748b;
      --line: #dbe3ee;
      --accent: #2563eb;
    }

    * {
      box-sizing: border-box;
    }

    body {
      margin: 0;
      font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
      background: var(--background);
      color: var(--text);
    }

    main {
      width: min(1400px, 94vw);
      margin: 32px auto 56px;
    }

    h1 {
      margin-bottom: 6px;
    }

    h2 {
      margin-top: 0;
    }

    .muted {
      color: var(--muted);
    }

    .cards {
      display: grid;
      grid-template-columns: repeat(auto-fit, minmax(190px, 1fr));
      gap: 16px;
      margin: 24px 0;
    }

    .card,
    section {
      background: var(--card);
      border: 1px solid var(--line);
      border-radius: 14px;
      box-shadow: 0 8px 24px rgba(15, 23, 42, 0.05);
    }

    .card {
      padding: 18px;
    }

    .card strong {
      display: block;
      margin-top: 5px;
      font-size: 1.8rem;
    }

    section {
      padding: 20px;
      margin-top: 20px;
    }

    .tables {
      display: grid;
      grid-template-columns: repeat(auto-fit, minmax(360px, 1fr));
      gap: 20px;
    }

    .table-wrap {
      overflow-x: auto;
    }

    table {
      width: 100%;
      border-collapse: collapse;
    }

    th,
    td {
      padding: 10px 12px;
      border-bottom: 1px solid var(--line);
      text-align: left;
      white-space: nowrap;
    }

    th {
      background: #f8fafc;
    }

    #map {
      height: 620px;
      border-radius: 10px;
    }

    .downloads a {
      display: inline-block;
      margin-right: 14px;
      color: var(--accent);
      font-weight: 650;
      text-decoration: none;
    }

    code {
      font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
    }
  </style>
</head>
<body>
<main>
  <h1>IP Traffic Analysis Report</h1>

  <div class="muted">
    Generated ${GENERATED_AT} from ${TOTAL_LOG_LINES} non-empty access-log entries.
  </div>

  <div class="cards">
    <div class="card">
      <span class="muted">Log entries</span>
      <strong>${TOTAL_LOG_LINES}</strong>
    </div>

    <div class="card">
      <span class="muted">Public-IP requests</span>
      <strong>${PUBLIC_REQUESTS}</strong>
    </div>

    <div class="card">
      <span class="muted">Unique public IPs</span>
      <strong>${UNIQUE_IP_COUNT}</strong>
    </div>

    <div class="card">
      <span class="muted">Countries</span>
      <strong>${COUNTRY_COUNT}</strong>
    </div>

    <div class="card">
      <span class="muted">Mapped IPs</span>
      <strong>${MAPPED_IP_COUNT}</strong>
    </div>

    <div class="card">
      <span class="muted">Proxy requests</span>
      <strong>${PROXY_REQUESTS}</strong>
    </div>
  </div>

  <section>
    <h2>Traffic map</h2>
    <p class="muted">
      Each marker represents one unique public IP address. Nearby markers are clustered.
    </p>
    <div id="map"></div>
  </section>

  <div class="tables">
    <section>
      <h2>Top countries</h2>
      <div class="table-wrap">
        <table>
          <thead>
            <tr>
              <th>Country</th>
              <th>Code</th>
              <th>Requests</th>
              <th>Unique IPs</th>
            </tr>
          </thead>
          <tbody>
            ${TOP_COUNTRIES_ROWS}
          </tbody>
        </table>
      </div>
    </section>

    <section>
      <h2>Top cities</h2>
      <div class="table-wrap">
        <table>
          <thead>
            <tr>
              <th>Location</th>
              <th>Requests</th>
              <th>Unique IPs</th>
            </tr>
          </thead>
          <tbody>
            ${TOP_CITIES_ROWS}
          </tbody>
        </table>
      </div>
    </section>
  </div>

  <section>
    <h2>Top IP addresses</h2>
    <div class="table-wrap">
      <table>
        <thead>
          <tr>
            <th>IP address</th>
            <th>Requests</th>
            <th>Country</th>
            <th>City</th>
            <th>ISP</th>
          </tr>
        </thead>
        <tbody>
          ${TOP_IP_ROWS}
        </tbody>
      </table>
    </div>
  </section>

  <section class="downloads">
    <h2>Downloads</h2>
    <a href="traffic-details.csv">Detailed CSV</a>
    <a href="traffic-data.json">Detailed JSON</a>
  </section>
</main>

<script src="https://unpkg.com/[email protected]/dist/leaflet.js"></script>
<script src="https://unpkg.com/[email protected]/dist/leaflet.markercluster.js"></script>

<script>
const map = L.map('map').setView([20, 0], 2);

L.tileLayer(
  'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
  {
    maxZoom: 18,
    attribution: '© OpenStreetMap contributors'
  }
).addTo(map);

const markers = L.markerClusterGroup();
const bounds = [];

function escapeHtml(value) {
  return String(value ?? '').replace(
    /[&<>'"]/g,
    function (character) {
      return {
        '&': '&',
        '<': '<',
        '>': '>',
        "'": ''',
        '"': '"'
      }[character];
    }
  );
}

fetch('traffic-data.json')
  .then(function (response) {
    if (!response.ok) {
      throw new Error('Unable to load traffic-data.json');
    }

    return response.json();
  })
  .then(function (trafficData) {
    trafficData.forEach(function (row) {
      if (
        typeof row.latitude !== 'number' ||
        typeof row.longitude !== 'number'
      ) {
        return;
      }

      const location = [
        row.city_name,
        row.region_name,
        row.country_name
      ].filter(Boolean).join(', ');

      const proxyText = row.is_proxy === true ? 'Yes' : 'No';

      const popup =
        '<strong>' + escapeHtml(row.ip) + '</strong><br>' +
        'Requests: ' + Number(row.request_count || 0) + '<br>' +
        'Location: ' + escapeHtml(location || 'Unknown') + '<br>' +
        'ISP: ' + escapeHtml(row.isp || '') + '<br>' +
        'Proxy: ' + proxyText;

      const marker = L.marker([
        row.latitude,
        row.longitude
      ]).bindPopup(popup);

      markers.addLayer(marker);
      bounds.push([row.latitude, row.longitude]);
    });

    map.addLayer(markers);

    if (bounds.length > 0) {
      map.fitBounds(bounds, {
        padding: [25, 25],
        maxZoom: 8
      });
    }
  })
  .catch(function (error) {
    document.getElementById('map').innerHTML =
      '<p style="padding: 20px">Map error: ' +
      escapeHtml(error.message) +
      '</p>';
  });
</script>
</body>
</html>
HTML

mv -f "$REPORT_TMP" "$REPORT_DIR/index.html"
chmod 0644 \
    "$REPORT_DIR/index.html" \
    "$REPORT_DIR/traffic-details.csv" \
    "$REPORT_DIR/traffic-data.json"

printf '\nReport created successfully:\n'
printf '  HTML: %s/index.html\n' "$REPORT_DIR"
printf '  CSV:  %s/traffic-details.csv\n' "$REPORT_DIR"
printf '  JSON: %s/traffic-data.json\n' "$REPORT_DIR"