From 6ce86768d4b2287c23754536e33fcf262578a0a5 Mon Sep 17 00:00:00 2001 From: hrbrmstr Date: Mon, 7 Sep 2026 10:09:24 -0400 Subject: chore: initial commit --- .sync.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 .sync.sh (limited to '.sync.sh') diff --git a/.sync.sh b/.sync.sh new file mode 100755 index 0000000..6ede186 --- /dev/null +++ b/.sync.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Pull all bulk-data files listed by pathwren.workers.dev into /data/pathwren. +set -euo pipefail +DEST=/data/pathwren +BASE=https://www.pathwren.workers.dev +cd "$DEST" +curl -fsS --retry 3 "$BASE/data/index.json" -o .index.json +jq -r '.entries[].url' .index.json | while read -r u; do + f=$(basename "$u") + curl -fsS --retry 3 -o "$f.part" "$u" + mv -f "$f.part" "$f" +done +# Prune files no longer in the index (never touch .index.json or this script). +find . -maxdepth 1 -type f -name '*.part' -delete +find . -maxdepth 1 -type f ! -name '.index.json' ! -name '.sync.sh' -printf '%f\n' | while read -r f; do + jq -e --arg f "$f" 'any(.entries[]; (.url | ltrimstr("/") | split("/")[-1]) == $f)' .index.json >/dev/null || rm -f -- "$f" +done -- cgit v1.2.3