fossil-archive: update to generate template and save logs

1 files changed, 39 insertions, 4 deletions

diff --git a/bin/fossil-archive b/bin/fossil-archive
index 6a244d6..b953d4c 100755
--- a/bin/fossil-archive
+++ b/bin/fossil-archive
@@ -8,20 +8,55 @@ set -eu
 
 readonly mepath="$(readlink -m $(dirname $0)/..)"
 
+starttime="$(date +%s)"
+ddate() { date -u "-d@$starttime" "$1"; }
 base_dir="$1"; shift
 base_url="$1"; shift
 
+# hacky thing to make sure url ends in /
+if ! echo "$base_url" | grep -q '/$'; then
+	base_url="${base_url}/"
+fi
+
 safeurl() { echo "$1" | sed 's,/,_,g'; }
 
 [ -d "$base_dir" ] || mkdir "$base_dir"
 
-item_id="$(safeurl "$(echo "$base_url" | sed -e 's,^http,,;s,^s,,;s,^://,,' -e 's,$,_bundles_'"$(date +%Y%m%d)"',')")"
+item_id="$(safeurl "$(echo "$base_url" | sed -e 's,^http,,;s,^s,,;s,^://,,')")_bundles_$(ddate '+%Y%m%d')"
 work_dir="${base_dir}/${item_id}"
 
-echo "making $item_id folder"
+echo "-- making $item_id folder --"
 mkdir "$work_dir"
 
 while IFS='' read -r url; do
-	fossil clone --unversioned -v --once --no-open "$url" "$work_dir/$(safeurl "$url").bundle" || true
-	cp "${mepath}/samples/fossil-bundle.dir-to-ia.config" "${work_dir}/dir-to-ia.config"
+	urlp="$work_dir/$(safeurl "$url")"
+	{
+		echo "$ fossil clone --unversioned -v --once --no-open $url $urlp.bundle --"
+		USER=archiveteam-codearchiver-runner fossil clone --unversioned -v --once --no-open "$url" "$urlp.bundle" || true
+	} | tee "$urlp.log"
 done
+
+echo "-- finished job for $base_url --"
+echo "-- generating dir-to-ia.conf (check it and write as .config) --"
+
+cat >"${work_dir}/.dir-to-ia.conf" <<EoC
+exit 1 # make user edit check this
+sha256=yes
+rm=no
+rmwait=yes
+sha1check=yes
+clobber=no
+derive=no
+
+# Custom options for ia-upload-stream, as an array of args; this can be used to choose the part size and concurrency, for example, using (--part-size 1G --concurrency 4).
+iauploadstreamopts=()
+
+# Item metadata (array with 'key:value' elements); the only mandatory variable with no default
+metadata=(
+	"collection:open_source_software"
+	"date:$(ddate '+%Y-%m-%d')"
+	"description:Fossil repos for repositories under $base_url as of $(ddate '+%Y-%m-%d %H:%M') UTC. Retrieved with $(fossil version|sed -e 's,^This is ,,' -e 's,^,(,;s,$,),'), see corresponding log files for command(s) ran"
+	"mediatype:software"
+	"title:Fossil wanderinghorse.net repository clones (2025-12-01)"
+)
+EoC