.sync.sh


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

#!/usr/bin/env bash
# Pull all bulk-data files listed by pathwren.workers.dev into /data/pathwren.
set -euo pipefail
DEST=/data/pathwren
BASE=https://www.pathwren.workers.dev
cd "$DEST"
curl -fsS --retry 3 "$BASE/data/index.json" -o .index.json
jq -r '.entries[].url' .index.json | while read -r u; do
  f=$(basename "$u")
  curl -fsS --retry 3 -o "$f.part" "$u"
  mv -f "$f.part" "$f"
done
# Prune files no longer in the index (never touch .index.json or this script).
find . -maxdepth 1 -type f -name '*.part' -delete
find . -maxdepth 1 -type f ! -name '.index.json' ! -name '.sync.sh' -printf '%f\n' | while read -r f; do
  jq -e --arg f "$f" 'any(.entries[]; (.url | ltrimstr("/") | split("/")[-1]) == $f)' .index.json >/dev/null || rm -f -- "$f"
done