1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
|
#!/usr/bin/env bash
# probe llms.txt for each domain in companies.csv (name,url,domain)
set -u
cd "$(dirname "$0")"
UA="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
mkdir -p llms
probe() {
local name="$1" url="$2" domain="$3"
local base="$domain"
local first="${domain%%.*}" rest="${domain#*.}"
if [[ "$rest" == *.* || "$rest" == co.uk ]]; then
case "$first" in www|www2|help|usa|cybersecurity) base="$rest" ;; esac
fi
local out="llms/${domain}.txt"
local code code2 bytes tried
code=$(curl -sL --compressed --max-time 25 -A "$UA" -o "$out" -w '%{http_code}' "https://${domain}/llms.txt" 2>/dev/null)
tried="https://${domain}/llms.txt"
if [[ "$code" != "200" ]]; then
code2=$(curl -sL --compressed --max-time 25 -A "$UA" -o "$out" -w '%{http_code}' "https://www.${base}/llms.txt" 2>/dev/null)
if [[ "$code2" == "200" ]]; then
tried="${tried}|https://www.${base}/llms.txt"
code="$code2"
fi
fi
bytes=$(wc -c < "$out" | tr -d ' ')
printf '"%s","%s","%s","%s","%s","%s"\n' "$name" "$domain" "$code" "$bytes" "$tried" "$out"
}
export -f probe
export UA
tail -n +2 companies.csv | awk -F',' '{print $1"\t"$2"\t"$3}' \
| while IFS=$'\t' read -r name url domain; do probe "$name" "$url" "$domain"; done > results_raw.csv
{
echo 'name,domain,http_code,bytes,urls_tried,file'
cat results_raw.csv
} > results.csv
echo "done: $(tail -n +2 results.csv | wc -l | tr -d ' ') rows"
|
