1 files changed, 17 insertions, 0 deletions
diff --git a/.sync.sh b/.sync.sh
new file mode 100755
index 0000000..6ede186
--- /dev/null
+++ b/.sync.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Pull all bulk-data files listed by pathwren.workers.dev into /data/pathwren.
+set -euo pipefail
+DEST=/data/pathwren
+BASE=https://www.pathwren.workers.dev
+cd "$DEST"
+curl -fsS --retry 3 "$BASE/data/index.json" -o .index.json
+jq -r '.entries[].url' .index.json | while read -r u; do
+  f=$(basename "$u")
+  curl -fsS --retry 3 -o "$f.part" "$u"
+  mv -f "$f.part" "$f"
+done
+# Prune files no longer in the index (never touch .index.json or this script).
+find . -maxdepth 1 -type f -name '*.part' -delete
+find . -maxdepth 1 -type f ! -name '.index.json' ! -name '.sync.sh' -printf '%f\n' | while read -r f; do
+  jq -e --arg f "$f" 'any(.entries[]; (.url | ltrimstr("/") | split("/")[-1]) == $f)' .index.json >/dev/null || rm -f -- "$f"
+done