add: cybersec-llmstxt

1 files changed, 39 insertions, 0 deletions

diff --git a/2026/cybersec-llmstxt/probe.sh b/2026/cybersec-llmstxt/probe.sh
new file mode 100755
index 0000000..1c4ac94
--- /dev/null
+++ b/2026/cybersec-llmstxt/probe.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# probe llms.txt for each domain in companies.csv (name,url,domain)
+set -u
+cd "$(dirname "$0")"
+UA="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+mkdir -p llms
+
+probe() {
+  local name="$1" url="$2" domain="$3"
+  local base="$domain"
+  local first="${domain%%.*}" rest="${domain#*.}"
+  if [[ "$rest" == *.* || "$rest" == co.uk ]]; then
+    case "$first" in www|www2|help|usa|cybersecurity) base="$rest" ;; esac
+  fi
+  local out="llms/${domain}.txt"
+  local code code2 bytes tried
+  code=$(curl -sL --compressed --max-time 25 -A "$UA" -o "$out" -w '%{http_code}' "https://${domain}/llms.txt" 2>/dev/null)
+  tried="https://${domain}/llms.txt"
+  if [[ "$code" != "200" ]]; then
+    code2=$(curl -sL --compressed --max-time 25 -A "$UA" -o "$out" -w '%{http_code}' "https://www.${base}/llms.txt" 2>/dev/null)
+    if [[ "$code2" == "200" ]]; then
+      tried="${tried}|https://www.${base}/llms.txt"
+      code="$code2"
+    fi
+  fi
+  bytes=$(wc -c < "$out" | tr -d ' ')
+  printf '"%s","%s","%s","%s","%s","%s"\n' "$name" "$domain" "$code" "$bytes" "$tried" "$out"
+}
+export -f probe
+export UA
+
+tail -n +2 companies.csv | awk -F',' '{print $1"\t"$2"\t"$3}' \
+  | while IFS=$'\t' read -r name url domain; do probe "$name" "$url" "$domain"; done > results_raw.csv
+
+{
+  echo 'name,domain,http_code,bytes,urls_tried,file'
+  cat results_raw.csv
+} > results.csv
+echo "done: $(tail -n +2 results.csv | wc -l | tr -d ' ') rows"