#!/usr/bin/env bash
# probe llms.txt for each domain in companies.csv (name,url,domain)
set -u
cd "$(dirname "$0")"
UA="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
mkdir -p llms

probe() {
  local name="$1" url="$2" domain="$3"
  local base="$domain"
  local first="${domain%%.*}" rest="${domain#*.}"
  if [[ "$rest" == *.* || "$rest" == co.uk ]]; then
    case "$first" in www|www2|help|usa|cybersecurity) base="$rest" ;; esac
  fi
  local out="llms/${domain}.txt"
  local code code2 bytes tried
  code=$(curl -sL --compressed --max-time 25 -A "$UA" -o "$out" -w '%{http_code}' "https://${domain}/llms.txt" 2>/dev/null)
  tried="https://${domain}/llms.txt"
  if [[ "$code" != "200" ]]; then
    code2=$(curl -sL --compressed --max-time 25 -A "$UA" -o "$out" -w '%{http_code}' "https://www.${base}/llms.txt" 2>/dev/null)
    if [[ "$code2" == "200" ]]; then
      tried="${tried}|https://www.${base}/llms.txt"
      code="$code2"
    fi
  fi
  bytes=$(wc -c < "$out" | tr -d ' ')
  printf '"%s","%s","%s","%s","%s","%s"\n' "$name" "$domain" "$code" "$bytes" "$tried" "$out"
}
export -f probe
export UA

tail -n +2 companies.csv | awk -F',' '{print $1"\t"$2"\t"$3}' \
  | while IFS=$'\t' read -r name url domain; do probe "$name" "$url" "$domain"; done > results_raw.csv

{
  echo 'name,domain,http_code,bytes,urls_tried,file'
  cat results_raw.csv
} > results.csv
echo "done: $(tail -n +2 results.csv | wc -l | tr -d ' ') rows"